Update target vz to use OAM descriptors (#837)

* JIRA WDT-522 - First pass at changing to OAM descriptors

* JIRA WDT-530 - Add domainHomeSourceType to resource file for -target vz output

* JIRA WDT-522 - Use OAM descriptor for Verrazzano target

* JIRA WDT-522 - Add runtime encryption secret to secret creation script

* JIRA WDT-522 - Remove obsolete model template

* JIRA WDT-522 - Minor corrections, backport changes to target wko

* JIRA WDT-522 - Add runtime encryption secret to target wko; provide additional comment

* JIRA WDT-522 - Minor corrections; allow multiple additional secrets

* JIRA WDT-522 - Use domain UID for namespace for target wko

* JIRA WDT-522 - Adjusted comment

Author: rakillen

core/src/main/python/wlsdeploy/tool/util/k8s_helper.py

@@ -1,5 +1,5 @@
 """
-Copyright (c) 2020, Oracle Corporation and/or its affiliates.
+Copyright (c) 2020, 2021, Oracle and/or its affiliates.
 Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
 
 Methods and constants for building Kubernetes resource files,
@@ -26,7 +26,16 @@ def get_domain_uid(domain_name):
     :param domain_name: the domain name to be checked
     :return: the domain UID
     """
-    result = domain_name.lower()
+    return get_dns_name(domain_name)
+
+
+def get_dns_name(name):
+    """
+    Return a DNS-1123 compatible name, with the pattern ^[a-z0-9-.]{1,253}$
+    :param name: the domain name to be converted
+    :return: the DNS-1123 compatible name
+    """
+    result = name.lower()
     # replace any disallowed character with hyphen
     result = re.sub('[^a-z0-9-.]', '-', result)
     return result

core/src/main/python/wlsdeploy/tool/util/targets/additional_output_helper.py

@@ -1,12 +1,13 @@
 """
-Copyright (c) 2020, Oracle Corporation and/or its affiliates.
+Copyright (c) 2020, 2021, Oracle and/or its affiliates.
 Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
 
 Methods for creating Kubernetes resource configuration files for Verrazzano.
 """
 from java.io import File
 
 from wlsdeploy.aliases.location_context import LocationContext
+from wlsdeploy.aliases.model_constants import APPLICATION
 from wlsdeploy.aliases.model_constants import CLUSTER
 from wlsdeploy.aliases.model_constants import DEFAULT_WLS_DOMAIN_NAME
 from wlsdeploy.aliases.model_constants import JDBC_DRIVER_PARAMS
@@ -27,21 +28,27 @@
 # substitution keys used in the templates
 ADDITIONAL_SECRET_NAME = 'additionalSecretName'
 ADDITIONAL_SECRETS = 'additionalSecrets'
+APPLICATIONS = 'applications'
+APPLICATION_NAME = 'applicationName'
+APPLICATION_PREFIX = 'applicationPrefix'
 CLUSTER_NAME = 'clusterName'
 CLUSTERS = 'clusters'
-DATABASE_CREDENTIALS = 'databaseCredentials'
-DATABASE_PREFIX = 'databasePrefix'
-DATABASES = 'databases'
+DATASOURCE_CREDENTIALS = 'datasourceCredentials'
+DATASOURCE_PREFIX = 'datasourcePrefix'
+DATASOURCES = 'datasources'
 DATASOURCE_NAME = 'datasourceName'
+DATASOURCE_URL = 'url'
 DOMAIN_NAME = 'domainName'
 DOMAIN_PREFIX = 'domainPrefix'
 DOMAIN_TYPE = 'domainType'
 DOMAIN_UID = 'domainUid'
-DS_URL = 'url'
 HAS_ADDITIONAL_SECRETS = 'hasAdditionalSecrets'
+HAS_APPLICATIONS = 'hasApplications'
 HAS_CLUSTERS = 'hasClusters'
-HAS_DATABASES = 'hasDatabases'
+HAS_DATASOURCES = 'hasDatasources'
+NAMESPACE = 'namespace'
 REPLICAS = 'replicas'
+RUNTIME_ENCRYPTION_SECRET = "runtimeEncryptionSecret"
 WEBLOGIC_CREDENTIALS_SECRET = 'webLogicCredentialsSecret'
 
 
@@ -104,13 +111,14 @@ def _build_template_hash(model, model_context, aliases, credential_injector):
     domain_name = dictionary_utils.get_element(model.get_model_topology(), NAME)
     if domain_name is None:
         domain_name = DEFAULT_WLS_DOMAIN_NAME
+    template_hash[DOMAIN_NAME] = domain_name
 
-    # domain UID, name and prefix must follow DNS-1123
+    # domain UID, prefix, and namespace must follow DNS-1123
 
     domain_uid = k8s_helper.get_domain_uid(domain_name)
     template_hash[DOMAIN_UID] = domain_uid
-    template_hash[DOMAIN_NAME] = domain_uid
     template_hash[DOMAIN_PREFIX] = domain_uid
+    template_hash[NAMESPACE] = domain_uid
 
     # secrets that should not be included in secrets section
     declared_secrets = []
@@ -121,6 +129,12 @@ def _build_template_hash(model, model_context, aliases, credential_injector):
     declared_secrets.append(admin_secret)
     template_hash[WEBLOGIC_CREDENTIALS_SECRET] = admin_secret
 
+    # runtime encryption secret
+
+    runtime_secret = domain_uid + target_configuration_helper.RUNTIME_ENCRYPTION_SECRET_SUFFIX
+    declared_secrets.append(runtime_secret)
+    template_hash[RUNTIME_ENCRYPTION_SECRET] = runtime_secret
+
     # configuration / model
     template_hash[DOMAIN_TYPE] = model_context.get_domain_type()
 
@@ -159,20 +173,39 @@ def _build_template_hash(model, model_context, aliases, credential_injector):
         url = dictionary_utils.get_element(driver_params, URL)
         if url is None:
             url = ''
-        database_hash[DS_URL] = url
+        database_hash[DATASOURCE_URL] = url
 
         # should change spaces to hyphens?
-        database_hash[DATABASE_PREFIX] = jdbc_name.lower()
+        database_hash[DATASOURCE_PREFIX] = k8s_helper.get_dns_name(jdbc_name)
 
         # get the name that matches secret
         location.add_name_token(name_token, jdbc_name)
         secret_name = target_configuration_helper.get_secret_name_for_location(location, domain_uid, aliases)
-        database_hash[DATABASE_CREDENTIALS] = secret_name
+        database_hash[DATASOURCE_CREDENTIALS] = secret_name
 
         databases.append(database_hash)
 
-    template_hash[DATABASES] = databases
-    template_hash[HAS_DATABASES] = len(databases) != 0
+    template_hash[DATASOURCES] = databases
+    template_hash[HAS_DATASOURCES] = len(databases) != 0
+
+    # applications
+
+    apps = []
+
+    applications = dictionary_utils.get_dictionary_element(model.get_model_app_deployments(), APPLICATION)
+    for app_name in applications:
+        app_hash = dict()
+        prefix = '/' + app_name
+
+        # get the prefix from the app descriptor?
+
+        app_hash[APPLICATION_NAME] = app_name
+        app_hash[APPLICATION_PREFIX] = prefix
+
+        apps.append(app_hash)
+
+    template_hash[APPLICATIONS] = apps
+    template_hash[HAS_APPLICATIONS] = len(apps) != 0
 
     # additional secrets - exclude admin
 

core/src/main/python/wlsdeploy/tool/util/targets/file_template_helper.py

@@ -1,5 +1,5 @@
 """
-Copyright (c) 2020, Oracle Corporation and/or its affiliates.
+Copyright (c) 2020, 2021 Oracle Corporation and/or its affiliates.
 Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
 
 Methods for template substitution.
@@ -41,7 +41,7 @@ def create_file_from_resource(resource_path, template_hash, output_file, excepti
         __logger.throwing(ex, class_name=__class_name, method_name=_method_name)
         raise ex
 
-    _create_file_from_stream(template_stream, template_hash, output_file, exception_type)
+    _create_file_from_stream(template_stream, template_hash, output_file)
 
 
 def create_file_from_file(file_path, template_hash, output_file, exception_type):
@@ -58,49 +58,87 @@ def create_file_from_file(file_path, template_hash, output_file, exception_type)
     try:
         template_stream = FileUtils.getFileAsStream(file_path)
         if template_stream is not None:
-            _create_file_from_stream(template_stream, template_hash, output_file, exception_type)
+            _create_file_from_stream(template_stream, template_hash, output_file)
     except (IOException, IllegalArgumentException), ie:
         ex = exception_helper.create_exception(exception_type, 'WLSDPLY-01666', file_path, ie)
         __logger.throwing(ex, class_name=__class_name, method_name=_method_name)
         raise ex
 
 
-def _create_file_from_stream(template_stream, template_hash, output_file, exception_type):
+def _create_file_from_stream(template_stream, template_hash, output_file):
     template_reader = BufferedReader(InputStreamReader(template_stream))
     file_writer = open(output_file.getPath(), "w")
 
-    current_block_key = None
+    block_key = None
     block_lines = []
 
-    more = True
-    while more:
+    line = ''
+    while line is not None:
         line = template_reader.readLine()
         if line is not None:
             block_start_key = _get_block_start_key(line)
-            block_end_key = _get_block_end_key(line)
 
-            # if this is a nested block start, continue and add without substitution
-            if (block_start_key is not None) and (current_block_key is None):
-                current_block_key = block_start_key
-                block_lines = []
+            # if inside a block, collect lines until end key is found, then process the block.
+            if block_key is not None:
+                block_end_key = _get_block_end_key(line)
+                if block_end_key == block_key:
+                    _process_block(block_key, block_lines, template_hash, file_writer)
+                    block_key = None
+                else:
+                    block_lines.append(line)
 
-            # if this is a nested block end, continue and add without substitution
-            elif (block_end_key == current_block_key) and (current_block_key is not None):
-                _write_block(current_block_key, block_lines, template_hash, file_writer)
-                current_block_key = None
+            # if this is a block start, begin collecting block lines
+            elif block_start_key is not None:
+                block_key = block_start_key
+                block_lines = []
 
+            # otherwise, substitute and write the line
             else:
                 line = _substitute_line(line, template_hash)
+                file_writer.write(line + "\n")
 
-                if current_block_key is not None:
-                    block_lines.append(line)
+    file_writer.close()
+
+
+def _process_block(block_key, template_lines, template_hash, file_writer):
+    value = dictionary_utils.get_element(template_hash, block_key)
+
+    if value is None:
+        return
+
+    if not isinstance(value, list):
+        value = [value]
+
+    for list_element in value:
+        nested_block_key = None
+        nested_block_lines = []
+
+        for line in template_lines:
+            block_start_key = _get_block_start_key(line)
+
+            # if inside a block, collect lines until end key is found,
+            # then process the block with an updated hash.
+            if nested_block_key is not None:
+                block_end_key = _get_block_end_key(line)
+                if block_end_key == nested_block_key:
+                    nested_hash = dict(template_hash)
+                    if isinstance(list_element, dict):
+                        nested_hash.update(list_element)
+                    _process_block(nested_block_key, nested_block_lines, nested_hash, file_writer)
+                    nested_block_key = None
                 else:
-                    file_writer.write(line + "\n")
+                    nested_block_lines.append(line)
 
-        else:
-            more = False
+            # if this is a block start, begin collecting block lines
+            elif block_start_key is not None:
+                nested_block_key = block_start_key
+                nested_block_lines = []
 
-    file_writer.close()
+            # otherwise, substitute and write the line
+            else:
+                if isinstance(list_element, dict):
+                    line = _substitute_line(line, list_element)
+                file_writer.write(line + "\n")
 
 
 def _get_block_start_key(line):
@@ -142,31 +180,3 @@ def _substitute_line(line, template_hash):
         if replacement is not None:
             line = line.replace(token, replacement)
     return line
-
-
-def _write_block(key, lines, template_hash, file_writer):
-    """
-    Write a block of lines to the file writer, making substitutions as necessary.
-    This method does not currently handle nested blocks.
-    :param key: the key of this block
-    :param lines: the lines to be output
-    :param template_hash: the parent hash
-    :param file_writer: to write the output
-    """
-    value = dictionary_utils.get_element(template_hash, key)
-
-    # skip block for value of False, None, or empty collection
-    if not value:
-        return
-
-    # if value is not a list, make it a list with one item
-    if not isinstance(value, list):
-        value = [value]
-
-    for list_element in value:
-        for line in lines:
-            # this does not account for nested blocks
-
-            if isinstance(list_element, dict):
-                line = _substitute_line(line, list_element)
-            file_writer.write(line + "\n")

core/src/main/python/wlsdeploy/util/target_configuration.py

@@ -1,5 +1,5 @@
 """
-Copyright (c) 2020, Oracle Corporation and/or its affiliates.
+Copyright (c) 2020, 2021, Oracle and/or its affiliates.
 Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
 """
 
@@ -84,6 +84,16 @@ def get_variable_injectors(self):
         """
         return dictionary_utils.get_dictionary_element(self.config_dictionary, 'variable_injectors')
 
+    def get_additional_secrets(self):
+        """
+        Return a list of secrets to be included in the create secrets script.
+        :return: a list of secrets
+        """
+        secrets = dictionary_utils.get_element(self.config_dictionary, 'additional_secrets')
+        if secrets is not None:
+            return secrets.split(',')
+        return []
+
     def uses_credential_secrets(self):
         """
         Determine if this configuration uses secrets to manage credentials.

core/src/main/python/wlsdeploy/util/target_configuration_helper.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2020, Oracle Corporation and/or its affiliates.
+# Copyright (c) 2020, 2021, Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
 #
 # Shared methods for using target environments (-target abc).
@@ -30,6 +30,9 @@
 WEBLOGIC_CREDENTIALS_SECRET_NAME = 'weblogic-credentials'
 WEBLOGIC_CREDENTIALS_SECRET_SUFFIX = '-' + WEBLOGIC_CREDENTIALS_SECRET_NAME
 
+RUNTIME_ENCRYPTION_SECRET_NAME = 'runtime-encryption-secret'
+RUNTIME_ENCRYPTION_SECRET_SUFFIX = '-' + RUNTIME_ENCRYPTION_SECRET_NAME
+
 # keys for secrets, such as "password" in "jdbc-mydatasource:password"
 SECRET_USERNAME_KEY = "username"
 SECRET_PASSWORD_KEY = "password"
@@ -108,7 +111,7 @@ def generate_k8s_script(model_context, token_dictionary, model_dictionary, excep
 
     domain_uid = k8s_helper.get_domain_uid(domain_name)
     comment = exception_helper.get_message("WLSDPLY-01665")
-    script_hash = {'domainUid': domain_uid, 'topComment': comment}
+    script_hash = {'domainUid': domain_uid, 'topComment': comment, 'namespace': domain_uid}
 
     # build a map of secret names (jdbc-generic1) to keys (username, password)
     secret_map = {}
@@ -142,6 +145,16 @@ def generate_k8s_script(model_context, token_dictionary, model_dictionary, excep
         else:
             paired_secrets.append(_build_secret_hash(secret_name, user_name, PASSWORD_TAG))
 
+    # add a secret with a specific comment for runtime encryption
+    target_config = model_context.get_target_configuration()
+    additional_secrets = target_config.get_additional_secrets()
+    if RUNTIME_ENCRYPTION_SECRET_NAME in additional_secrets:
+        runtime_hash = _build_secret_hash(RUNTIME_ENCRYPTION_SECRET_NAME, None, PASSWORD_TAG)
+        message1 = exception_helper.get_message("WLSDPLY-01671", PASSWORD_TAG)
+        message2 = exception_helper.get_message("WLSDPLY-01672")
+        runtime_hash['comments'] = [{'comment': message1}, {'comment': message2}]
+        secrets.append(runtime_hash)
+
     script_hash['secrets'] = secrets
     script_hash['pairedSecrets'] = paired_secrets
     script_hash['longMessage'] = exception_helper.get_message('WLSDPLY-01667', '${LONG_SECRETS_COUNT}')
@@ -272,7 +285,7 @@ def _build_secret_hash(secret_name, user, password):
     """
     if user:
         message = exception_helper.get_message("WLSDPLY-01664", USER_TAG, PASSWORD_TAG, secret_name)
-        return {'secretName': secret_name, 'user': user, 'password': password, 'comment': message}
+        return {'secretName': secret_name, 'user': user, 'password': password, 'comments': [{'comment': message}]}
     else:
         message = exception_helper.get_message("WLSDPLY-01663", PASSWORD_TAG, secret_name)
-        return {'secretName': secret_name, 'password': password, 'comment': message}
+        return {'secretName': secret_name, 'password': password, 'comments': [{'comment': message}]}