registry: Fix a bug with parsing gitlab registries, fix downloading from

public gitlab repositories

misc: Cleanup, download from the correct version when downloading from gitlab.

Author: nouwaarom

src/clib-install.c

@@ -371,9 +371,7 @@ int main(int argc, char *argv[]) {
     memset(prefix, 0, path_max);
     realpath(opts.prefix, prefix);
     unsigned long int size = strlen(prefix) + 1;
-    opts.prefix = malloc(size);
-    memset((void *) opts.prefix, 0, size);
-    memcpy((void *) opts.prefix, prefix, size);
+    opts.prefix = strndup(prefix, size);
   }
 
   clib_cache_init(CLIB_PACKAGE_CACHE_TIME);
@@ -407,6 +405,7 @@ int main(int argc, char *argv[]) {
 
   curl_global_cleanup();
   clib_package_cleanup();
+  clib_package_free(root_package);
 
   command_free(&program);
   return code;

src/clib-update.c

@@ -16,6 +16,7 @@
 #include "logger/logger.h"
 #include "parson/parson.h"
 #include "str-replace/str-replace.h"
+#include "strdup/strdup.h"
 #include "version.h"
 #include <clib-package-installer.h>
 #include <curl/curl.h>
@@ -339,9 +340,7 @@ int main(int argc, char *argv[]) {
     memset(prefix, 0, path_max);
     realpath(package_opts.prefix, prefix);
     unsigned long int size = strlen(prefix) + 1;
-    package_opts.prefix = malloc(size);
-    memset((void *) package_opts.prefix, 0, size);
-    memcpy((void *) package_opts.prefix, prefix, size);
+    package_opts.prefix = strndup(prefix, size);
   }
 
   clib_cache_init(CLIB_PACKAGE_CACHE_TIME);

src/clib-upgrade.c

@@ -16,6 +16,7 @@
 #include "logger/logger.h"
 #include "parson/parson.h"
 #include "str-replace/str-replace.h"
+#include "strdup/strdup.h"
 #include "tempdir/tempdir.h"
 #include "version.h"
 #include <asprintf/asprintf.h>
@@ -232,9 +233,7 @@ int main(int argc, char *argv[]) {
     memset(prefix, 0, path_max);
     realpath(opts.prefix, prefix);
     unsigned long int size = strlen(prefix) + 1;
-    opts.prefix = malloc(size);
-    memset((void *) opts.prefix, 0, size);
-    memcpy((void *) opts.prefix, prefix, size);
+    opts.prefix = strndup(prefix, size);
   }
 
   clib_cache_init(CLIB_PACKAGE_CACHE_TIME);

src/common/clib-package-installer.c

@@ -98,7 +98,7 @@ static inline int install_packages(list_t *list, const char *dir, int verbose) {
     registry_package_ptr_t package_info = registry_manager_find_package(registries, package_id);
     if (!package_info) {
       logger_error("package-installer", "Package %s not found in any registry.", package_id);
-      return -1;
+      goto loop_cleanup;
     }
 
     pkg = clib_package_new_from_slug_and_url(slug, registry_package_get_href(package_info), verbose);
@@ -112,6 +112,7 @@ static inline int install_packages(list_t *list, const char *dir, int verbose) {
     error = 0;
 
   loop_cleanup:
+    free(package_id);
     if (slug)
       free(slug);
     if (error) {
@@ -300,7 +301,6 @@ int clib_package_install(clib_package_t *pkg, const char *dir, int verbose) {
   char *package_json = NULL;
   char *pkg_dir = NULL;
   char *command = NULL;
-  int pending = 0;
   int rc = 0;
   int i = 0;
 
@@ -485,41 +485,25 @@ int clib_package_install(clib_package_t *pkg, const char *dir, int verbose) {
 
   iterator = list_iterator_new(pkg->src, LIST_HEAD);
   list_node_t *source;
-  repository_file_handle_t *handles = malloc(pkg->src->len * sizeof(repository_file_handle_t));
+  repository_file_handle_t *handles = malloc(max * sizeof(repository_file_handle_t));
+  char* package_id = clib_package_get_id(pkg->author, pkg->repo_name);
+  // TODO, refactor this.
   while ((source = list_iterator_next(iterator))) {
-    handles[i] = repository_download_package_file(pkg->url, clib_package_get_id(pkg->author, pkg->repo_name), pkg->version, source->val, pkg_dir);
-
+    handles[i] = repository_download_package_file(pkg->url, package_id, pkg->version, source->val, pkg_dir);
     if (handles[i] == NULL) {
       list_iterator_destroy(iterator);
       iterator = NULL;
       rc = -1;
       goto cleanup;
     }
 
-#if defined(__unix__) || (defined(__APPLE__) && defined(__MACH__))
-    struct timespec ts = {0, 1000 * 1000 * 10};
-    nanosleep(&ts, NULL);
-#endif
-
 #ifdef HAVE_PTHREADS
-    if (i < 0) {
-      i = 0;
-    }
-
-    (void) pending++;
-
     if (i < max) {
-      (void) i++;
+      i++;
     } else {
       while (--i >= 0) {
         repository_file_finish_download(handles[i]);
         repository_file_free(handles[i]);
-        (void) pending--;
-
-#if defined(__unix__) || (defined(__APPLE__) && defined(__MACH__))
-        struct timespec ts = {0, 1000 * 1000 * 10};
-        nanosleep(&ts, NULL);
-#endif
       }
     }
 #endif
@@ -528,8 +512,6 @@ int clib_package_install(clib_package_t *pkg, const char *dir, int verbose) {
 #ifdef HAVE_PTHREADS
   while (--i >= 0) {
     repository_file_finish_download(handles[i]);
-
-    (void) pending--;
     repository_file_free(handles[i]);
   }
 #endif

src/registry/gitlab-registry.c

@@ -8,8 +8,17 @@
 #include "http-get/http-get.h"
 #include "registry-internal.h"
 #include <curl/curl.h>
-#include <string.h>
 #include <strdup/strdup.h>
+#include <string.h>
+
+static char *string_split(char *in, char sep) {
+  char *next_sep = strchr(in, sep);
+  if (next_sep == NULL) {
+    return next_sep;
+  }
+  *next_sep = '\0';
+  return next_sep + sizeof(char);
+}
 
 /**
  * Parse a list of packages from the given `html`
@@ -21,7 +30,7 @@ static list_t *gitlab_registry_parse(const char *hostname, const char *html) {
   char *input = strdup(html);
   char *line = input;
   char *category = NULL;
-  while ((line = strtok(line, "\n"))) {
+  while ((line = string_split(line, '\n'))) {
     char *dash_position = strstr(line, "-");
     // The line starts with a dash, so we expect a package.
     if (dash_position != NULL && dash_position - line < 4) {
@@ -58,14 +67,14 @@ static list_t *gitlab_registry_parse(const char *hostname, const char *html) {
 list_t *gitlab_registry_fetch(const char *url, const char *hostname, const char *secret) {
   http_get_response_t *res;
   if (secret == NULL) {
-    return NULL;
+    res = http_get(url, NULL, 0);
+  } else {
+    char *key = "PRIVATE-TOKEN";
+    unsigned int size = strlen(key) + strlen(secret) + 2;
+    char *authentication_header = malloc(size);
+    snprintf(authentication_header, size, "%s:%s", key, secret);
+    res = http_get(url, &authentication_header, 1);
   }
-
-  char *key = "PRIVATE-TOKEN";
-  unsigned int size = strlen(key) + strlen(secret) + 2;
-  char *authentication_header = malloc(size);
-  snprintf(authentication_header, size, "%s:%s", key, secret);
-  res = http_get(url, &authentication_header, 1);
   if (!res->ok) {
     return NULL;
   }

src/registry/registry-manager.c

@@ -42,7 +42,7 @@ void registry_manager_fetch_registries(registries_t registries) {
   registry_ptr_t reg;
   while ((reg = registry_iterator_next(it))) {
       if (!registry_fetch(reg)) {
-      printf("REGISTRY: could not list packages from. %s\n", registry_get_url(reg));
+      printf("REGISTRY: could not list packages from. %s secret is: %s\n", registry_get_url(reg), registry_get_secret(reg));
     }
   }
   registry_iterator_destroy(it);

src/registry/registry.c

@@ -96,6 +96,10 @@ const char *registry_get_url(registry_ptr_t registry) {
   return registry->url;
 }
 
+const char* registry_get_secret(registry_ptr_t registry) {
+  return registry->secret;
+}
+
 bool registry_fetch(registry_ptr_t registry) {
   switch (registry->type) {
   case REGISTRY_TYPE_GITLAB:

src/registry/registry.h

@@ -37,10 +37,17 @@ bool registry_fetch(registry_ptr_t registry);
 /**
  * Get the url for the registry
  * @param registry
- * @return
+ * @return the url
  */
 const char* registry_get_url(registry_ptr_t registry);
 
+/**
+ * Get the secret for this registry
+ * @param registry
+ * @return the secret or NULL if there is no secret.
+ */
+const char* registry_get_secret(registry_ptr_t registry);
+
 /**
  * An iterator through the packages in the registry.
  */

src/repository/gitlab-repository.c

@@ -9,17 +9,20 @@
 #include <stdlib.h>
 #include <string.h>
 #include <url/url.h>
+#include "str-replace/str-replace.h"
 
-#define GITLAB_API_V4_URL "https://%s/api/v4%s/repository/files/%s/raw?ref=master"
+#define GITLAB_API_V4_URL "https://%s/api/v4%s/repository/files/%s/raw?ref=%s"
 
 // GET :hostname/api/v4/projects/:id/repository/files/:file_path/raw
-char* gitlab_repository_get_url_for_file(const char*package_url, const char* slug, const char* version, const char *file, const char* secret) {
+char *gitlab_repository_get_url_for_file(const char *package_url, const char *slug, const char *version, const char *file, const char *secret) {
   url_data_t *parsed = url_parse(package_url);
 
-  int size = strlen(parsed->hostname) + strlen(parsed->pathname) + strlen(file) + 64;
+  char* encoded_filename = str_replace(file, "/", "%2F");
+
+  size_t size = strlen(parsed->hostname) + strlen(parsed->pathname) + strlen(encoded_filename) + strlen(GITLAB_API_V4_URL) + strlen(version) + 1;
   char *url = malloc(size);
   if (url) {
-    snprintf(url, size, GITLAB_API_V4_URL, parsed->hostname, parsed->pathname, file);
+    snprintf(url, size, GITLAB_API_V4_URL, parsed->hostname, parsed->pathname, encoded_filename, version);
   }
 
   url_free(parsed);

src/repository/repository.c

@@ -29,7 +29,7 @@ static debug_t _debugger;
   })
 
 struct repository_file_t {
-  const char *url;
+  char *url;
   const char *dir;
   const char *file;
   const char *secret;
@@ -87,7 +87,7 @@ http_get_response_t *repository_fetch_package_manifest(const char *package_url,
   char *manifest_url = repository_create_url_for_file(package_url, package_id, version, manifest_file, secret);
 
   http_get_response_t *res;
-  if (strstr(package_url, "gitlab") != NULL) {
+  if (secret && strstr(package_url, "gitlab") != NULL) {
     char *key = "PRIVATE-TOKEN";
     unsigned int size = strlen(key) + strlen(secret) + 2;
     char *authentication_header = malloc(size);
@@ -98,6 +98,8 @@ http_get_response_t *repository_fetch_package_manifest(const char *package_url,
     res = http_get_shared(manifest_url, clib_package_curl_share, NULL, 0);
   }
 
+  free(manifest_url);
+
   return res;
 }
 
@@ -116,7 +118,11 @@ repository_file_handle_t repository_download_package_file(const char *package_ur
 
 void repository_file_finish_download(repository_file_handle_t file) {
   void *rc;
-  pthread_join(file->thread, &rc);
+  int success = pthread_join(file->thread, &rc);
+  if (success != 0) {
+    printf("Failed to join thread.\n");
+  }
+  free(rc);
 }
 
 void repository_file_free(repository_file_handle_t file) {
@@ -133,8 +139,6 @@ static int fetch_package_file_work(const char *url, const char *dir, const char
     return 1;
   }
 
-  _debug("file URL: %s", url);
-
   if (!(path = path_join(dir, basename(file)))) {
     rc = 1;
     goto cleanup;
@@ -145,14 +149,14 @@ static int fetch_package_file_work(const char *url, const char *dir, const char
 #endif
 
   if (package_opts.force || -1 == fs_exists(path)) {
-    _debug("repository", "fetching %s", url);
+    _debug("fetching %s", url);
     fflush(stdout);
 
 #ifdef HAVE_PTHREADS
     pthread_mutex_unlock(&mutex);
 #endif
 
-    if (strstr(url, "gitlab") != NULL) {
+    if (secret && strstr(url, "gitlab") != NULL) {
       char *key = "PRIVATE-TOKEN";
       unsigned int size = strlen(key) + strlen(secret) + 2;
       char *authentication_header = malloc(size);
@@ -186,7 +190,7 @@ static int fetch_package_file_work(const char *url, const char *dir, const char
 #ifdef HAVE_PTHREADS
       pthread_mutex_lock(&mutex);
 #endif
-      _debug("repository", "saved %s", path);
+      _debug("saved %s", path);
       fflush(stdout);
 #ifdef HAVE_PTHREADS
       pthread_mutex_unlock(&mutex);