# -------------------------------------------------------------------
# Creation of a defined CRAN snapshot to be made available on 
# https://cran.intiquan.com 
#
# Run script from its location on the server!
#
# 1) Run from Treysa or similar 
# 2) Connect to "web_cran" and "projects" 
# 3) Run script 
# -------------------------------------------------------------------
#
# Details:
# - This script should be located on the "external server" in the folder web_cran
# - The script is "developed" in the IQDESKTOP git repository in 03_INST_SCRIPTS/CRAN_SNAPSHOT
# - The script is a further development of the take_CRAN_snapshot.R script from Henning. The
#   aim of the update was to remove the dependency on IQRtools and make it executable on
#   basel/riehen.


folder_proj <- "/mnt/iqserver/projects"
folder_cran <- "/mnt/iqserver/web_cran"

# Auxiliary function to create folders only if they don't exist (from IQRtools)
aux_mkdir <- function (pathdir) suppressWarnings(if (!file.exists(pathdir)) dir.create(pathdir, recursive = TRUE))

# Auxiliary function to read file
aux_fileread <- function (filename) {
    fid <- file(filename, open = "r")
    text <- readLines(fid)
    close(fid)
    return(text)
}

# Check folders mounted
if (system(paste("mountpoint -q", folder_proj)) != 0) stop("Mount projects!")
if (system(paste("mountpoint -q", folder_cran)) != 0) stop("Mount web_cran!")

# Get and check path (assume script is in the current path)
if (!file.exists("take_CRAN_snapshot_new.R")) stop("Check path!")
scriptPath <- getwd()

# Date of snapshot defined by date it is taken (Basel time and date)
# Format: 2022-04-01
SNAPSHOT_DATE <- format(Sys.time(), "%Y-%m-%d")
cat("Date of CRAN-snapshot to be taken:", SNAPSHOT_DATE,"\n")

# Other definitions
NCORES <- parallel::detectCores()
CRANurl <- "https://cran.r-project.org/src/contrib/"
CRANurl_OLDER <- "https://cran.r-project.org/src/contrib/Older/"
SNAPSHOTpath <- paste0("snapshot/",SNAPSHOT_DATE,"/src/contrib/")
SNAPSHOT_OLDERpath <- paste0(SNAPSHOTpath,"Older/")
cat("  Stored in:", SNAPSHOTpath,"\n")


# Do not allow to overwrite if already exists 
# (requires manual deletion if re-download desired) 
if (file.exists(SNAPSHOTpath)) stop("Snapshot already present")

# Create folders for snapshots to download
aux_mkdir(SNAPSHOTpath)
aux_mkdir(SNAPSHOT_OLDERpath)

# ----------------------------------
# Main Repo
# ----------------------------------

# Download CRAN package page as text file
download.file(url = CRANurl,destfile = "content.txt",method = "libcurl",quiet = TRUE)

# Parse the content file to get all names of package files
content <- aux_fileread("content.txt")
content <- content[grepl(".tar.gz",content)]
unlink("content.txt")

# Parse package file names
m <- gregexpr('<a(.*?)</a>',content)
matches <- unlist(regmatches(content,m))
m <- gregexpr('<a href=\"(.*?)\">',matches)
matches <- unlist(regmatches(matches,m))
matches <- gsub('<a href=\"|\">',"",matches)
#matches
cat("  Number of \"Standard\" packages to download:", length(m),"\n")

#matches <- matches[1:5]

# Download all source packages
chunk_size <- 50
matches_list <- split(matches, ceiling(seq_along(matches) / chunk_size))

parallel::mclapply(matches_list, function (myfiles) {
  download.file(
    url = paste0(CRANurl, myfiles), 
    destfile = paste0(SNAPSHOTpath, myfiles),
    quiet = TRUE,
    method = "libcurl")
},mc.cores = ifelse(.Platform$OS.type=="windows",1,NCORES),mc.preschedule = FALSE)

# ----------------------------------
# Older R Version Repo
# ----------------------------------

# Download CRAN package page as text file
download.file(url = CRANurl_OLDER,destfile = "content.txt",method = "libcurl",quiet = TRUE)

# Parse the content file to get all names of package files
content <- aux_fileread("content.txt")
content <- content[grepl(".tar.gz",content)]
unlink("content.txt")

# Parse package file names
m <- gregexpr('<a(.*?)</a>',content)
matches <- unlist(regmatches(content,m))
m <- gregexpr('<a href=\"(.*?)\">',matches)
matches <- unlist(regmatches(matches,m))
matches <- gsub('<a href=\"|\">',"",matches)
#matches
cat("  Number of \"Older\" packages to download:", length(m),"\n")

#matches <- matches[1:2]

# Download all source packages
parallel::mclapply(seq_along(matches), function (k) {
  download.file(url = paste0(CRANurl_OLDER,matches[k]),destfile = paste0(SNAPSHOT_OLDERpath,matches[k]),quiet = TRUE,method = "libcurl")
},mc.cores = ifelse(.Platform$OS.type=="windows",1,NCORES),mc.preschedule = FALSE)

# ----------------------------------
# Generate repo information and end
# ----------------------------------

# Write the PACKAGES files
setwd(SNAPSHOTpath)
cat("  Writing package information ... \n")
tools::write_PACKAGES(".",type = "source")

# Change folder back
setwd(scriptPath)

# ----------------------------------
# Ensure backup of repo
# ----------------------------------

cat("  Copying to backup location ... \n")
aux_mkdir(file.path(folder_proj, "Software Development/cran_intiquan/snapshot", SNAPSHOT_DATE))
command <- paste0("cp -r ", folder_cran, "/snapshot/", SNAPSHOT_DATE, " ", folder_proj, "/Software\\ Development/cran_intiquan/snapshot")
system(command)

# Final message
cat("Downloading CRAN snapshot completed!","\n")