# -------------------------------------------------------------------
# Creation of a defined CRAN snapshot to be made available on 
# https://cran.intiquan.com 
#
# Run script from its location on the server!
#
# 1) Run from Treysa or similar 
# 2) Connect to "web_cran" and "projects" 
# 3) Run script 
# -------------------------------------------------------------------

# Check folders mounted
if (length(list.files("~/PROJECTS/MOUNT/web_cran")) == 0) stop("Mount web_cran!")
if (length(list.files("~/PROJECTS/MOUNT/projects")) == 0) stop("Mount projects!")

# Get and check path (assume script is in the current path)
if (!file.exists("take_CRAN_snapshot.R")) stop("Check path!")
scriptPath <- getwd()

# Date of snapshot defined by date it is taken (Basel time and date)
# Format: 2022-04-01
SNAPSHOT_DATE <- format(Sys.time(), "%Y-%m-%d")
cat("Date of CRAN-snapshot to be taken:", SNAPSHOT_DATE,"\n")

# Other definitions
NCORES <- 50
CRANurl <- "https://cran.r-project.org/src/contrib/"
CRANurl_OLDER <- "https://cran.r-project.org/src/contrib/Older/"
SNAPSHOTpath <- paste0("snapshot/",SNAPSHOT_DATE,"/src/contrib/")
SNAPSHOT_OLDERpath <- paste0(SNAPSHOTpath,"Older/")
cat("  Stored in:", SNAPSHOTpath,"\n")


# Do not allow to overwrite if already exists 
# (requires manual deletion if re-download desired) 
if (file.exists(SNAPSHOTpath)) stop("Snapshot already present")

# Create folders for snapshots to download
IQRtools::aux_mkdir(SNAPSHOTpath)
IQRtools::aux_mkdir(SNAPSHOT_OLDERpath)

# ----------------------------------
# Main Repo
# ----------------------------------

# Download CRAN package page as text file
download.file(url = CRANurl,destfile = "content.txt",method = "libcurl",quiet = TRUE)

# Parse the content file to get all names of package files
content <- IQRtools::aux_fileread("content.txt",collapserows = FALSE)
content <- content[grepl(".tar.gz",content)]
unlink("content.txt")

# Parse package file names
m <- gregexpr('<a(.*?)</a>',content)
matches <- unlist(regmatches(content,m))
m <- gregexpr('<a href=\"(.*?)\">',matches)
matches <- unlist(regmatches(matches,m))
matches <- gsub('<a href=\"|\">',"",matches)
#matches
cat("  Number of \"Standard\" packages to download:", length(m),"\n")

#matches <- matches[1:5]

# Download all source packages
parallel::mclapply(seq_along(matches), function (k) {
  download.file(url = paste0(CRANurl,matches[k]),destfile = paste0(SNAPSHOTpath,matches[k]),quiet = TRUE,method = "libcurl")
},mc.cores = ifelse(.Platform$OS.type=="windows",1,NCORES),mc.preschedule = FALSE)


# ----------------------------------
# Older R Version Repo
# ----------------------------------

# Download CRAN package page as text file
download.file(url = CRANurl_OLDER,destfile = "content.txt",method = "libcurl",quiet = TRUE)

# Parse the content file to get all names of package files
content <- IQRtools::aux_fileread("content.txt",collapserows = FALSE)
content <- content[grepl(".tar.gz",content)]
unlink("content.txt")

# Parse package file names
m <- gregexpr('<a(.*?)</a>',content)
matches <- unlist(regmatches(content,m))
m <- gregexpr('<a href=\"(.*?)\">',matches)
matches <- unlist(regmatches(matches,m))
matches <- gsub('<a href=\"|\">',"",matches)
#matches
cat("  Number of \"Older\" packages to download:", length(m),"\n")

#matches <- matches[1:2]

# Download all source packages
parallel::mclapply(seq_along(matches), function (k) {
  download.file(url = paste0(CRANurl_OLDER,matches[k]),destfile = paste0(SNAPSHOT_OLDERpath,matches[k]),quiet = TRUE,method = "libcurl")
},mc.cores = ifelse(.Platform$OS.type=="windows",1,NCORES),mc.preschedule = FALSE)

# ----------------------------------
# Generate repo information and end
# ----------------------------------

# Write the PACKAGES files
setwd(SNAPSHOTpath)
cat("  Writing package information ... \n")
tools::write_PACKAGES(".",type = "source")

# Change folder back
setwd(scriptPath)

# ----------------------------------
# Ensure backup of repo
# ----------------------------------

cat("  Copying to backup location ... \n")
IQRtools::aux_mkdir(paste0("~/PROJECTS/MOUNT/projects/Software Development/cran_intiquan/snapshot/",SNAPSHOT_DATE))
command <- paste0("cp -r ~/PROJECTS/MOUNT/web_cran/snapshot/",SNAPSHOT_DATE," ~/PROJECTS/MOUNT/projects/Software\\ Development/cran_intiquan/snapshot")
system(command)

# Final message
cat("Downloading CRAN snapshot completed!","\n")