diff --git a/Source/MantisSourceGitBasePlugin.class.php b/Source/MantisSourceGitBasePlugin.class.php
index 068e83135..bb444d0f6 100644
--- a/Source/MantisSourceGitBasePlugin.class.php
+++ b/Source/MantisSourceGitBasePlugin.class.php
@@ -46,6 +46,7 @@ abstract class MantisSourceGitBasePlugin extends MantisSourcePlugin
* Error constants
*/
const ERROR_INVALID_BRANCH = 'invalid_branch';
+ const ERROR_INVALID_DATE = 'invalid_date';
/**
* Define plugin's Error strings
@@ -54,6 +55,7 @@ abstract class MantisSourceGitBasePlugin extends MantisSourcePlugin
public function errors() {
$t_errors_list = array(
self::ERROR_INVALID_BRANCH,
+ self::ERROR_INVALID_DATE,
);
foreach( $t_errors_list as $t_error ) {
@@ -73,6 +75,19 @@ protected function is_branch_valid( $p_branch )
return (bool)preg_match( $this->valid_branch_regex, $p_branch );
}
+ /**
+ * Determines if given string name is a valid regex.
+ * @param string $p_regex regex to validate
+ * @return bool True if valid
+ */
+ protected function is_regex_branch_valid( $p_regex )
+ {
+ # Trick to compile the regex
+ # see ReturnValues of the official doc https://www.php.net/manual/en/function.preg-match.php
+ return (0 === preg_match( $p_regex, "" ));
+ }
+
+
/**
* Triggers an error if the branch is invalid
* @param string $p_branch Branch name to validate
@@ -89,15 +104,20 @@ protected function ensure_branch_valid( $p_branch )
/**
* Validates a comma-delimited list of git branches.
* Triggers an ERROR_INVALID_BRANCH if one of the branches is invalid
- * @param string $p_list Comma-delimited list of branch names (or '*')
+ * @param string $p_list Comma-delimited list of branch names, or a regex, or '*'
* @return void
*/
protected function validate_branch_list( $p_list )
{
+ #Case '*'
if( $p_list == '*' ) {
return;
}
-
+ #Case regex
+ if( preg_match( "/^\/.+\/[a-z]*$/i", $p_list ) ) {
+ return;
+ }
+ #Case list of validi git branches
foreach( explode( ',', $p_list ) as $t_branch ) {
$this->ensure_branch_valid( trim( $t_branch ) );
}
@@ -114,6 +134,21 @@ protected function get_default_primary_branches() {
return $t_value;
}
+ /**
+ * Validates a date
+ * Triggers an ERROR_INVALID_DATE if date is not valid
+ * @return void
+ */
+ protected function validate_date($p_date) {
+ if (empty($p_date)) {
+ return;
+ }
+ if (! (bool)strtotime($p_date)) {
+ error_parameters( $p_date );
+ plugin_error( self::ERROR_INVALID_DATE );
+ }
+}
+
/**
* Output form elements for configuration options.
*/
diff --git a/Source/lang/strings_english.txt b/Source/lang/strings_english.txt
index 6973b1fb1..3c08408c3 100644
--- a/Source/lang/strings_english.txt
+++ b/Source/lang/strings_english.txt
@@ -138,7 +138,9 @@ $s_plugin_Source_changeset_removed = 'Changeset removed';
$s_plugin_Source_ensure_delete = 'Do you really want to delete the "%s" repository and all of its history?';
$s_plugin_Source_ensure_import_full = 'This will import to a new copy of your repository, and then destroy the old repository data, and may require use of shell access. Do you wish to continue?';
+$s_plugin_Source_ensure_import__full_since = 'This will import to a new copy of your repository since "%s", and then destroy the old repository data, and may require use of shell access. Do you wish to continue?';
$s_plugin_Source_ensure_import_latest = 'This will attempt to import recent data for your repository, and may require use of shell access. Do you wish to continue?';
+$s_plugin_Source_ensure_import_latest_since = 'This will attempt to import since "%s" for yout repository, and may require use of shell access. Do you wish to continue?';
$s_plugin_Source_import_results = 'Import Results';
$s_plugin_Source_import_stats = 'Imported %s changesets, %s files, and %s bug references.';
$s_plugin_Source_import_repo_error = 'Import process produced an error.';
@@ -150,6 +152,7 @@ $s_plugin_Source_invalid_changeset = 'Changeset information could not be loaded'
$s_plugin_Source_import_latest_failed = 'Repository latest data importing failed.';
$s_plugin_Source_import_full_failed = 'Full repository data importing failed.';
+$s_plugin_Source_import_since_failed = 'Repository "since date" importing failed.';
$s_plugin_Source_changeset_column_title = 'C';
diff --git a/Source/lang/strings_french.txt b/Source/lang/strings_french.txt
index 398de63fc..03a6dbc79 100644
--- a/Source/lang/strings_french.txt
+++ b/Source/lang/strings_french.txt
@@ -70,6 +70,7 @@ $s_plugin_Source_back_repo = 'Retour au dépôt';
$s_plugin_Source_back_changeset = 'Retour au jeu de changements';
$s_plugin_Source_import_full = 'Tout importer';
$s_plugin_Source_import_latest = 'Importer les dernières données';
+$s_plugin_Source_import_since = 'Importer depuis le...';
$s_plugin_Source_related_changesets = 'Jeux de changements liés';
$s_plugin_Source_affected_issues = 'Demandes affectées';
$s_plugin_Source_attach_to_issue = 'Attacher des demandes :';
@@ -134,7 +135,9 @@ $s_plugin_Source_changeset_removed = 'Jeu de changement détaché';
$s_plugin_Source_ensure_delete = 'Voulez-vous vraiment supprimer le dépôt "%s" et tout son historique ?';
$s_plugin_Source_ensure_import_full = 'Ceci va importer une nouvelle copie de votre dépôt, puis détruira les vieilles données de dépôt et pourrait avoir besoin d\'un accès au shell. Voulez-vous continuer ?';
+$s_plugin_Source_ensure_import_full_since = 'Ceci va importer une nouvelle copie de votre dépôt depuis le "%s", puis détruira les vieilles données de dépôt et pourrait avoir besoin d\'un accès au shell. Voulez-vous continuer ?';
$s_plugin_Source_ensure_import_latest = 'Ceci va tenter d\'importer les données récentes dans votre dépôt et pourrait avoir besoin d\'un accès au shell. Voulez-vous continuer ?';
+$s_plugin_Source_ensure_import_latest_since = 'Cevi va tenter d\'importer les données depuis le "%s" dans votre dépôt et pourrait avoir besoin d\'un accès au shell. Voulez-vous continuer ?';
$s_plugin_Source_import_results = 'Résultats de l\'import';
$s_plugin_Source_import_stats = '%s jeux de changements importés, %s fichiers, et %s références de demande.';
$s_plugin_Source_import_repo_error = 'Le processus d\'import a rencontré une erreur.';
diff --git a/Source/pages/repo_import_full.php b/Source/pages/repo_import_full.php
index 93c104dc3..3b602b506 100644
--- a/Source/pages/repo_import_full.php
+++ b/Source/pages/repo_import_full.php
@@ -6,12 +6,18 @@
form_security_validate( 'plugin_Source_repo_import_full' );
access_ensure_global_level( plugin_config_get( 'manage_threshold' ) );
-$f_repo_id = gpc_get_string( 'id' );
+$f_repo_id = gpc_get_int( 'id' );
$t_repo = SourceRepo::load( $f_repo_id );
$t_vcs = SourceVCS::repo( $t_repo );
-helper_ensure_confirmed( plugin_lang_get( 'ensure_import_full' ), plugin_lang_get( 'import_full' ) );
+if (empty( $t_repo->info['hub_oldest_commit_date'])){
+ helper_ensure_confirmed( plugin_lang_get( 'ensure_import_full' ), plugin_lang_get( 'import_full' ) );
+}else{
+ $t_date_since = $t_repo->info['hub_oldest_commit_date'];
+ helper_ensure_confirmed( sprintf( plugin_lang_get( 'ensure_import_full_since' ), $t_date_since), plugin_lang_get( 'import_full' ) );
+}
+
helper_begin_long_process();
layout_page_header( plugin_lang_get( 'title' ) );
diff --git a/Source/pages/repo_import_latest.php b/Source/pages/repo_import_latest.php
index a84cf82d5..b553e01cd 100644
--- a/Source/pages/repo_import_latest.php
+++ b/Source/pages/repo_import_latest.php
@@ -7,17 +7,23 @@
form_security_validate( 'plugin_Source_repo_import_latest' );
access_ensure_global_level( plugin_config_get( 'manage_threshold' ) );
-helper_ensure_confirmed( plugin_lang_get( 'ensure_import_latest' ), plugin_lang_get( 'import_latest' ) );
-$f_repo_id = strtolower( gpc_get_string( 'id' ) );
+$f_repo_id = gpc_get_int( 'id' );
$t_repo_id = (int) $f_repo_id;
$t_repos = array( SourceRepo::load( $t_repo_id ) );
-
$t_repo = array_shift( $t_repos );
$t_vcs = SourceVCS::repo( $t_repo );
-$t_repo->pre_stats = $t_repo->stats();
+$t_pre_stats = $t_repo->stats();
+
+if ( empty( $t_repo->info['hub_oldest_commit_date'] ) ) {
+ helper_ensure_confirmed( plugin_lang_get( 'ensure_import_latest' ), plugin_lang_get( 'import_latest' ) );
+}else{
+ $t_date_since = $t_repo->info['hub_oldest_commit_date'];
+ helper_ensure_confirmed( sprintf(plugin_lang_get( 'ensure_import_latest_since' ), $t_date_since), plugin_lang_get( 'import_latest' ) );
+}
+
layout_page_header( plugin_lang_get( 'title' ) );
layout_page_begin();
@@ -39,7 +45,7 @@
import_error = false;
+$t_import_error = false;
while( true ) {
# import the next batch of changesets
@@ -47,7 +53,7 @@
# check for errors
if ( !is_array( $t_changesets ) ) {
- $t_repo->import_error = true;
+ $t_import_error = true;
break;
}
@@ -59,21 +65,21 @@
Source_Process_Changesets( $t_changesets );
}
-$t_repo->post_stats = $t_repo->stats();
+$t_post_stats = $t_repo->stats();
?>
| name ) ?> |
import_error ) {
+if ( $t_import_error) {
echo plugin_lang_get( 'import_latest_failed' ), '
';
}
-$t_stats = $t_repo->post_stats;
-$t_stats['changesets'] -= $t_repo->pre_stats['changesets'];
-$t_stats['files'] -= $t_repo->pre_stats['files'];
-$t_stats['bugs'] -= $t_repo->pre_stats['bugs'];
+$t_stats = $t_post_stats;
+$t_stats['changesets'] -= $t_pre_stats['changesets'];
+$t_stats['files'] -= $t_pre_stats['files'];
+$t_stats['bugs'] -= $t_pre_stats['bugs'];
echo sprintf( plugin_lang_get( 'import_stats' ), $t_stats['changesets'], $t_stats['files'], $t_stats['bugs'] );
?>
diff --git a/Source/pages/repo_manage_page.php b/Source/pages/repo_manage_page.php
index 1bf08a9bf..947a45088 100644
--- a/Source/pages/repo_manage_page.php
+++ b/Source/pages/repo_manage_page.php
@@ -177,12 +177,12 @@ function convert_to_key_value( $p_array ) {
+
-
+
diff --git a/SourceGitlab/SourceGitlab.php b/SourceGitlab/SourceGitlab.php
index 42fd0718b..263b36a31 100644
--- a/SourceGitlab/SourceGitlab.php
+++ b/SourceGitlab/SourceGitlab.php
@@ -94,6 +94,7 @@ public function update_repo_form( $p_repo ) {
$t_hub_repoid = null;
$t_hub_reponame = null;
$t_hub_app_secret = null;
+ $t_hub_oldest_commit_date = null;
if ( isset( $p_repo->info['hub_root'] ) ) {
$t_hub_root = $p_repo->info['hub_root'];
@@ -112,6 +113,9 @@ public function update_repo_form( $p_repo ) {
} else {
$t_master_branch = $this->get_default_primary_branches();
}
+ if ( isset( $p_repo->info['hub_oldest_commit_date'] ) ) {
+ $t_hub_oldest_commit_date = $p_repo->info['hub_oldest_commit_date'];
+ }
?>
|
|
@@ -182,6 +186,18 @@ public function update_repo_form( $p_repo ) {
/>
|
+
+ |
+
+ |
+
+
+ |
+
info['hub_repoid'] = $f_hub_repoid;
$p_repo->info['master_branch'] = $f_master_branch;
+
+ # Check date format
+ $t_hub_oldest_commit_date = gpc_get_string( 'hub_oldest_commit_date' );
+ $this->validate_date($t_hub_oldest_commit_date);
+
+ $p_repo->info['hub_oldest_commit_date'] = $t_hub_oldest_commit_date;
+
return $p_repo;
}
@@ -300,54 +323,91 @@ public function commit( $p_repo, $p_data ) {
return $this->import_commits( $p_repo, $t_commits, $t_branch );
}
- public function import_full( $p_repo ) {
- echo '';
+ protected function build_gitlab_apis($p_repo, $t_branch) {
+
+ $t_repoid = $p_repo->info['hub_repoid'];
+
+ /*
+ * Because Gitlab will return a pageg result (only the 20 first branches)
+ * The request for '*' should be reworked
+ */
+ if ( $t_branch === "*" ) {
+ return array( "projects/$t_repoid/repository/branches/");
+ }
+
+ if ( preg_match( "/^\/.+\/[a-z]*$/i", $t_branch )) /* is a regex ? */
+ {
+ return array("projects/$t_repoid/repository/branches/?regex=$t_branch");
+ }
+
+ $gitlab_url_by_name = function( $branch ) use ($t_repoid) {
+ $branch_name = urlencode($branch);
+ return "projects/$t_repoid/repository/branches/$branch_name/";
+ };
+
+ $t_branches = array_map( 'trim', explode( ',', $t_branch ) );
+ return array_map( $gitlab_url_by_name, $t_branches);
+ }
+
+
+ public function import_full( $p_repo) {
+ static $counter=0;
+ $counter = $counter + 1;
+ echo "Pass #$counter: \n";
$t_branch = $p_repo->info['master_branch'];
if ( is_blank( $t_branch ) ) {
$t_branch = $this->get_default_primary_branches();
}
- # if we're not allowed everything, populate an array of what we are allowed
- if( $t_branch != '*' ) {
- $t_branches_allowed = array_map( 'trim', explode( ',', $t_branch ) );
+ # Always pull back only interested branches
+ $t_api_names = $this->build_gitlab_apis($p_repo, $t_branch);
+
+ $t_uris = array();
+ foreach( $t_api_names as $t_api_name)
+ {
+ array_push($t_uris, $this->api_uri( $p_repo, $t_api_name));
}
- # Always pull back full list of repos
- $t_repoid = $p_repo->info['hub_repoid'];
- $t_uri = $this->api_uri( $p_repo, "projects/$t_repoid/repository/branches" );
-
- $t_member = null;
- $t_json = json_url( $t_uri, $t_member );
- if( $t_json === null ) {
- echo "Could not retrieve data from GitLab at '$t_uri'. Make sure your ";
- print_link(
- plugin_page( 'repo_update_page', null, 'Source' )
- . "&id=$p_repo->id",
- 'repository settings'
- );
- echo " are correct.";
- echo '';
- return array();
- }
+ $t_json = array();
+ try {
- $t_branches = array();
- foreach( $t_json as $t_branch ) {
- if( empty( $t_branches_allowed ) || in_array( $t_branch->name, $t_branches_allowed ) ) {
- $t_branches[] = $t_branch;
+ foreach ($t_uris as $t_uri)
+ {
+ $t_member = null;
+ #print_r($t_uri);
+ $t_json_tmp = json_url( $t_uri, $t_member );
+ #print_r($t_json_tmp);
+ if( $t_json_tmp === null ) {
+ echo "Could not retrieve data from GitLab at '$t_uri'. Make sure your ";
+ print_link(
+ plugin_page( 'repo_update_page', null, 'Source' )
+ . "&id=$p_repo->id",
+ 'repository settings'
+ );
+ echo " are correct.";
+ echo '';
+ return array();
}
+ array_push( $t_json, $t_json_tmp);
}
+ } catch (Exception $e) {
+ echo 'Caught exception: ', $e->getMessage(), "\n";
+ echo '';
+ return array();
+ }
+ #print_r($t_json);
$t_changesets = array();
$t_changeset_table = plugin_table( 'changeset', 'Source' );
- foreach( $t_branches as $t_branch ) {
+ foreach( $t_json as $t_branch ) {
$t_query = "SELECT parent FROM $t_changeset_table
WHERE repo_id=" . db_param() . ' AND branch=' . db_param() .
' ORDER BY timestamp';
$t_result = db_query( $t_query, array( $p_repo->id, $t_branch->name ), 1 );
-
+ echo "Process branch '$t_branch->name':\n";
$t_commits = array( $t_branch->commit->id );
if ( db_num_rows( $t_result ) > 0 ) {
$t_parent = db_result( $t_result );
@@ -355,9 +415,10 @@ public function import_full( $p_repo ) {
if ( !empty( $t_parent ) ) {
$t_commits[] = $t_parent;
- echo "Parents not empty";
+ echo "Parents not empty - ";
+ } else {
+ echo "Parents empty";
}
- echo "Parents empty";
}
$t_changesets = array_merge( $t_changesets, $this->import_commits( $p_repo, $t_commits, $t_branch->name ) );
@@ -414,37 +475,43 @@ public function import_commits( $p_repo, $p_commit_ids, $p_branch='' ) {
private function json_commit_changeset( $p_repo, $p_json, $p_branch='' ) {
echo "processing $p_json->id ... ";
- if ( !SourceChangeset::exists( $p_repo->id, $p_json->id ) ) {
- # Message will be replaced by title in gitlab version earlier than 7.2
- $t_message = ( !property_exists( $p_json, 'message' ) )
- ? $p_json->title
- : $p_json->message;
- $t_changeset = new SourceChangeset(
- $p_repo->id,
- $p_json->id,
- $p_branch,
- $p_json->created_at,
- $p_json->author_name,
- $t_message
- );
-
- $t_parents = array();
- foreach( $p_json->parent_ids as $t_parent ) {
- $t_parents[] = $t_parent;
- }
- if( $t_parents ) {
- $t_changeset->parent = $t_parents[0];
- }
-
- $t_changeset->author_email = $p_json->author_email;
- $t_changeset->save();
-
- echo "saved.\n";
- return array( $t_changeset, $t_parents );
- } else {
+ if ( SourceChangeset::exists( $p_repo->id, $p_json->id ) ) {
echo "already exists.\n";
return array( null, array() );
}
+
+ $t_oldest_commit_date = $p_repo->info['hub_oldest_commit_date'];
+ if ( !empty( $t_oldest_commit_date ) && new DateTime( $p_json->created_at ) < new DateTime( $t_oldest_commit_date ) ) {
+ echo "commit before the oldest_commit_date.\n";
+ return array( null, array() );
+ }
+
+ # Message will be replaced by title in gitlab version earlier than 7.2
+ $t_message = ( !property_exists( $p_json, 'message' ) )
+ ? $p_json->title
+ : $p_json->message;
+ $t_changeset = new SourceChangeset(
+ $p_repo->id,
+ $p_json->id,
+ $p_branch,
+ $p_json->created_at,
+ $p_json->author_name,
+ $t_message
+ );
+
+ $t_parents = array();
+ foreach( $p_json->parent_ids as $t_parent ) {
+ $t_parents[] = $t_parent;
+ }
+ if( $t_parents ) {
+ $t_changeset->parent = $t_parents[0];
+ }
+
+ $t_changeset->author_email = $p_json->author_email;
+ $t_changeset->save();
+
+ echo "saved.\n";
+ return array( $t_changeset, $t_parents );
}
public static function url_post( $p_url, $p_post_data ) {
diff --git a/SourceGitlab/lang/strings_english.txt b/SourceGitlab/lang/strings_english.txt
index 71d36fd75..7e4dce9f4 100644
--- a/SourceGitlab/lang/strings_english.txt
+++ b/SourceGitlab/lang/strings_english.txt
@@ -14,6 +14,7 @@ $s_plugin_SourceGitlab_hub_repoid = 'GitLab Repository ID
Name & Namespace with no leading or trailing slashes (namespace/project)';
$s_plugin_SourceGitlab_hub_app_secret = 'GitLab API Key
api key for mantis user (continuous user)';
$s_plugin_SourceGitlab_master_branch = 'Allowed Branches
(comma-separated list, regular expression with /.../ or *)';
+$s_plugin_SourceGitlab_hub_oldest_commit_date = 'Import since (optional)
Oldest commit will be ignored, leave empty to import all commits';
$s_plugin_SourceGitlab_import_limit = 'Import Limit
at most this many commits per import action (0: unlimited)';
$s_plugin_SourceGitlab_error_api = 'GitLab API error (%1$s)';