Merge pull request #2686 from NginxProxyManager/develop

v2.9.20

Author: jc21

.version

@@ -1 +1 @@
-2.9.19
+2.9.20

Jenkinsfile

@@ -1,3 +1,9 @@
+import groovy.transform.Field
+
+@Field
+def shOutput = ""
+def buildxPushTags = ""
+
 pipeline {
 	agent {
 		label 'docker-multiarch'
@@ -16,6 +22,8 @@ pipeline {
 		COMPOSE_FILE               = 'docker/docker-compose.ci.yml'
 		COMPOSE_INTERACTIVE_NO_CLI = 1
 		BUILDX_NAME                = "${COMPOSE_PROJECT_NAME}"
+		DOCS_BUCKET                = 'jc21-npm-site'
+		DOCS_CDN                   = 'EN1G6DEWZUTDT'
 	}
 	stages {
 		stage('Environment') {
@@ -26,7 +34,7 @@ pipeline {
 					}
 					steps {
 						script {
-							env.BUILDX_PUSH_TAGS = "-t docker.io/jc21/${IMAGE}:${BUILD_VERSION} -t docker.io/jc21/${IMAGE}:${MAJOR_VERSION} -t docker.io/jc21/${IMAGE}:latest"
+							buildxPushTags = "-t docker.io/jc21/${IMAGE}:${BUILD_VERSION} -t docker.io/jc21/${IMAGE}:${MAJOR_VERSION} -t docker.io/jc21/${IMAGE}:latest"
 						}
 					}
 				}
@@ -39,7 +47,7 @@ pipeline {
 					steps {
 						script {
 							// Defaults to the Branch name, which is applies to all branches AND pr's
-							env.BUILDX_PUSH_TAGS = "-t docker.io/jc21/${IMAGE}:github-${BRANCH_LOWER}"
+							buildxPushTags = "-t docker.io/jc21/${IMAGE}:github-${BRANCH_LOWER}"
 						}
 					}
 				}
@@ -54,35 +62,28 @@ pipeline {
 				}
 			}
 		}
-		stage('Frontend') {
+		stage('Build and Test') {
 			steps {
-				sh './scripts/frontend-build'
+				script {
+					// Frontend and Backend
+					def shStatusCode = sh(label: 'Checking and Building', returnStatus: true, script: '''
+						set -e
+						./scripts/ci/frontend-build > ${WORKSPACE}/tmp-sh-build 2>&1
+						./scripts/ci/test-and-build > ${WORKSPACE}/tmp-sh-build 2>&1
+					''')
+					shOutput = readFile "${env.WORKSPACE}/tmp-sh-build"
+					if (shStatusCode != 0) {
+						error "${shOutput}"
+					}
+				}
 			}
-		}
-		stage('Backend') {
-			steps {
-				echo 'Checking Syntax ...'
-				sh 'docker pull nginxproxymanager/nginx-full:certbot-node'
-				// See: https://github.com/yarnpkg/yarn/issues/3254
-				sh '''docker run --rm \\
-					-v "$(pwd)/backend:/app" \\
-					-v "$(pwd)/global:/app/global" \\
-					-w /app \\
-					nginxproxymanager/nginx-full:certbot-node \\
-					sh -c "yarn install && yarn eslint . && rm -rf node_modules"
-				'''
-
-				echo 'Docker Build ...'
-				sh '''docker build --pull --no-cache --squash --compress \\
-					-t "${IMAGE}:ci-${BUILD_NUMBER}" \\
-					-f docker/Dockerfile \\
-					--build-arg TARGETPLATFORM=linux/amd64 \\
-					--build-arg BUILDPLATFORM=linux/amd64 \\
-					--build-arg BUILD_VERSION="${BUILD_VERSION}" \\
-					--build-arg BUILD_COMMIT="${BUILD_COMMIT}" \\
-					--build-arg BUILD_DATE="$(date '+%Y-%m-%d %T %Z')" \\
-					.
-				'''
+			post {
+				always {
+					sh 'rm -f ${WORKSPACE}/tmp-sh-build'
+				}
+				failure {
+					npmGithubPrComment("CI Error:\n\n```\n${shOutput}\n```", true)
+				}
 			}
 		}
 		stage('Integration Tests Sqlite') {
@@ -164,10 +165,8 @@ pipeline {
 			}
 			steps {
 				withCredentials([usernamePassword(credentialsId: 'jc21-dockerhub', passwordVariable: 'dpass', usernameVariable: 'duser')]) {
-					// Docker Login
-					sh "docker login -u '${duser}' -p '${dpass}'"
-					// Buildx with push from cache
-					sh "./scripts/buildx --push ${BUILDX_PUSH_TAGS}"
+					sh 'docker login -u "${duser}" -p "${dpass}"'
+					sh "./scripts/buildx --push ${buildxPushTags}"
 				}
 			}
 		}
@@ -181,26 +180,7 @@ pipeline {
 				}
 			}
 			steps {
-				withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'npm-s3-docs', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) {
-					sh """docker run --rm \\
-						--name \${COMPOSE_PROJECT_NAME}-docs-upload \\
-						-e S3_BUCKET=jc21-npm-site \\
-						-e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \\
-						-e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \\
-						-v \$(pwd):/app \\
-						-w /app \\
-						jc21/ci-tools \\
-						scripts/docs-upload /app/docs/.vuepress/dist/
-					"""
-
-					sh """docker run --rm \\
-						--name \${COMPOSE_PROJECT_NAME}-docs-invalidate \\
-						-e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \\
-						-e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \\
-						jc21/ci-tools \\
-						aws cloudfront create-invalidation --distribution-id EN1G6DEWZUTDT --paths '/*'
-					"""
-				}
+				npmDocsRelease("$DOCS_BUCKET", "$DOCS_CDN")
 			}
 		}
 		stage('PR Comment') {
@@ -214,7 +194,7 @@ pipeline {
 			}
 			steps {
 				script {
-					def comment = pullRequest.comment("This is an automated message from CI:\n\nDocker Image for build ${BUILD_NUMBER} is available on [DockerHub](https://cloud.docker.com/repository/docker/jc21/${IMAGE}) as `jc21/${IMAGE}:github-${BRANCH_LOWER}`\n\n**Note:** ensure you backup your NPM instance before testing this PR image! Especially if this PR contains database changes.")
+					npmGithubPrComment("Docker Image for build ${BUILD_NUMBER} is available on [DockerHub](https://cloud.docker.com/repository/docker/jc21/${IMAGE}) as `jc21/${IMAGE}:github-${BRANCH_LOWER}`\n\n**Note:** ensure you backup your NPM instance before testing this PR image! Especially if this PR contains database changes.", true)
 				}
 			}
 		}

backend/internal/access-list.js

@@ -218,7 +218,7 @@ const internalAccessList = {
 				// re-fetch with expansions
 				return internalAccessList.get(access, {
 					id:     data.id,
-					expand: ['owner', 'items', 'clients', 'proxy_hosts.access_list.[clients,items]']
+					expand: ['owner', 'items', 'clients', 'proxy_hosts.[certificate,access_list.[clients,items]]']
 				}, true /* <- skip masking */);
 			})
 			.then((row) => {
@@ -256,7 +256,7 @@ const internalAccessList = {
 					.joinRaw('LEFT JOIN `proxy_host` ON `proxy_host`.`access_list_id` = `access_list`.`id` AND `proxy_host`.`is_deleted` = 0')
 					.where('access_list.is_deleted', 0)
 					.andWhere('access_list.id', data.id)
-					.allowEager('[owner,items,clients,proxy_hosts.[*, access_list.[clients,items]]]')
+					.allowEager('[owner,items,clients,proxy_hosts.[certificate,access_list.[clients,items]]]')
 					.omit(['access_list.is_deleted'])
 					.first();
 
@@ -507,7 +507,7 @@ const internalAccessList = {
 								if (typeof item.password !== 'undefined' && item.password.length) {
 									logger.info('Adding: ' + item.username);
 
-									utils.exec('/usr/bin/htpasswd -b "' + htpasswd_file + '" "' + item.username + '" "' + item.password + '"')
+									utils.execFile('/usr/bin/htpasswd', ['-b', htpasswd_file, item.username, item.password])
 										.then((/*result*/) => {
 											next();
 										})

backend/internal/certificate.js

@@ -874,7 +874,8 @@ const internalCertificate = {
 		// Escape single quotes and backslashes
 		const escapedCredentials = certificate.meta.dns_provider_credentials.replaceAll('\'', '\\\'').replaceAll('\\', '\\\\');
 		const credentialsCmd     = 'mkdir -p /etc/letsencrypt/credentials 2> /dev/null; echo \'' + escapedCredentials + '\' > \'' + credentialsLocation + '\' && chmod 600 \'' + credentialsLocation + '\'';
-		const prepareCmd         = 'pip install ' + dns_plugin.package_name + (dns_plugin.version_requirement || '') + ' ' + dns_plugin.dependencies;
+		// we call `. /opt/certbot/bin/activate` (`.` is alternative to `source` in dash) to access certbot venv
+		let prepareCmd = '. /opt/certbot/bin/activate && pip install ' + dns_plugin.package_name + (dns_plugin.version_requirement || '') + ' ' + dns_plugin.dependencies + ' && deactivate';
 
 		// Whether the plugin has a --<name>-credentials argument
 		const hasConfigArg = certificate.meta.dns_provider !== 'route53';

backend/internal/token.js

@@ -24,7 +24,7 @@ module.exports = {
 
 		return userModel
 			.query()
-			.where('email', data.identity)
+			.where('email', data.identity.toLowerCase().trim())
 			.andWhere('is_deleted', 0)
 			.andWhere('is_disabled', 0)
 			.first()

backend/lib/utils.js

@@ -1,4 +1,5 @@
-const exec = require('child_process').exec;
+const exec     = require('child_process').exec;
+const execFile = require('child_process').execFile;
 
 module.exports = {
 
@@ -16,5 +17,21 @@ module.exports = {
 				}
 			});
 		});
+	},
+
+	/**
+	 * @param   {Array} cmd
+	 * @returns {Promise}
+	 */
+	execFile: function (cmd) {
+		return new Promise((resolve, reject) => {
+			execFile(cmd, function (err, stdout, /*stderr*/) {
+				if (err && typeof err === 'object') {
+					reject(err);
+				} else {
+					resolve(stdout.trim());
+				}
+			});
+		});
 	}
 };

backend/package.json

@@ -11,13 +11,13 @@
 		"body-parser": "^1.19.0",
 		"compression": "^1.7.4",
 		"config": "^3.3.1",
-		"express": "^4.17.1",
+		"express": "^4.17.3",
 		"express-fileupload": "^1.1.9",
 		"gravatar": "^1.8.0",
 		"json-schema-ref-parser": "^8.0.0",
-		"jsonwebtoken": "^8.5.1",
-		"knex": "^0.20.13",
-		"liquidjs": "^9.11.10",
+		"jsonwebtoken": "^9.0.0",
+		"knex": "^2.4.0",
+		"liquidjs": "^10.0.0",
 		"lodash": "^4.17.21",
 		"moment": "^2.29.4",
 		"mysql": "^2.18.1",

backend/setup.js

@@ -174,13 +174,13 @@ const setupCertbotPlugins = () => {
 
 				certificates.map(function (certificate) {
 					if (certificate.meta && certificate.meta.dns_challenge === true) {
-						const dns_plugin          = dns_plugins[certificate.meta.dns_provider];
-						const packages_to_install = `${dns_plugin.package_name}${dns_plugin.version_requirement || ''} ${dns_plugin.dependencies}`;
+						const dns_plugin = dns_plugins[certificate.meta.dns_provider];
 
+						const packages_to_install = `${dns_plugin.package_name}${dns_plugin.version_requirement || ''} ${dns_plugin.dependencies}`;
 						if (plugins.indexOf(packages_to_install) === -1) plugins.push(packages_to_install);
 
 						// Make sure credentials file exists
-						const credentials_loc = '/etc/letsencrypt/credentials/credentials-' + certificate.id; 
+						const credentials_loc = '/etc/letsencrypt/credentials/credentials-' + certificate.id;
 						// Escape single quotes and backslashes
 						const escapedCredentials = certificate.meta.dns_provider_credentials.replaceAll('\'', '\\\'').replaceAll('\\', '\\\\');
 						const credentials_cmd    = '[ -f \'' + credentials_loc + '\' ] || { mkdir -p /etc/letsencrypt/credentials 2> /dev/null; echo \'' + escapedCredentials + '\' > \'' + credentials_loc + '\' && chmod 600 \'' + credentials_loc + '\'; }';
@@ -189,14 +189,14 @@ const setupCertbotPlugins = () => {
 				});
 
 				if (plugins.length) {
-					const install_cmd = 'pip install ' + plugins.join(' ');
+					const install_cmd = '. /opt/certbot/bin/activate && pip install ' + plugins.join(' ') + ' && deactivate';
 					promises.push(utils.exec(install_cmd));
 				}
 
 				if (promises.length) {
 					return Promise.all(promises)
-						.then(() => { 
-							logger.info('Added Certbot plugins ' + plugins.join(', ')); 
+						.then(() => {
+							logger.info('Added Certbot plugins ' + plugins.join(', '));
 						});
 				}
 			}