Added support to create 2019.11.21 version of Global Table (#27)

* Added support to create 2019 11 21 version of Global Table

* Returned the lighter version of lodash.get back and updated package-lock.json README.md files

* Added additional tips about v2 setup to README.md file

Co-authored-by: Oleksandr Kononenko <o.kononenko@spd-ukraine.com>

Author: alexandelphi

README.md

@@ -21,19 +21,40 @@ plugins:
 ```yaml
 custom:
   globalTables:
+    version: v1 # optional, default is 'v1' (2017.11.29), please use 'v2' for (2019.11.21) version creation
     regions: # list of regions in which you want to set up global tables
       - region-1
       - region-2
     createStack: false # optional flag, when set to false will not deploy the stack in new region(s) and will create the tables using AWS SDK.
+                       # if you use 'createStack: true' with 'version: v2', please add 'Condition' rule to your dynamodb to create it in the main region only,
+                       # other regions are going to be replicated automatically from the main region.
 ```
 
-_NOTE_: When creating global tables with `createStack: false`, any update the source table config is not replicated to global tables.
-
+_NOTE_:
+1. When creating global tables with `createStack: false`, any update the source table config is not replicated to global tables.   
+2. `version` field is backward compatible and not required (the field can be absent).   
+If you want to use Global Table (Version 2019.11.21), please use `version: v2`.   
+Also, we don't recommend using `v2` over `v1` in your existing project. The plugin doesn't support updating from `v1` to `v2`.   
+More details about Global Tables you can find in the following link: [AWS DynamoDB Global Tables](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/GlobalTables.html)
+3. Here is an example of using conditions, by default it's optional, but it's required for `createStack: true` with `version: v2` setup:
+```
+Conditions:
+  RegionUSEast1: !Equals [ !Ref "AWS::Region", us-east-1 ]
+
+MyDynamoDBTable:
+    Condition: RegionUSEast1
+    Type: 'AWS::DynamoDB::Table'
+    DeletionPolicy: Retain
+    Properties:
+    ....
+```
 ## Revisions
 * 2.0.0
   - Updated the package to deploy the service stack in the new region(s) by default
   - Added support to setup auto-scaling on global tables when not using create stack feature.
   - Added unit-tests
 * 2.1.0
   - Added support for PAY_PER_REQUEST billing mode for `createStack: true` mode
+* 3.1.0
+  - Added support to create a new version of Global Table (Version 2019.11.21)
 

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "serverless-create-global-dynamodb-table",
-  "version": "3.0.0",
+  "version": "3.1.0",
   "description": "serverless plugin that would create global dynamodb tables for specified tables",
   "main": "src/index.js",
   "scripts": {
@@ -34,7 +34,7 @@
     "sinon": "^7.4.2"
   },
   "dependencies": {
-    "aws-sdk": "^2.224.1",
+    "aws-sdk": "^2.804.0",
     "chalk": "^2.3.2",
     "lodash.get": "^4.4.2"
   },

package.json

@@ -184,17 +184,29 @@ const createNewTableAndSetScalingPolicy = async function createNewTableAndSetSca
  * @param {string} region AWS region in which source table exists
  * @param {Array} newRegions List of regions in which global tables needs to be created
  * @param {string} tableName Dynamodb table name
+ * @param {string} version It's version of global table needs to be setup
  * @param {Object} cli Serverless cli object
  * @returns {Object} List of regions in which gloabl table needs to be created and flag indicating
  * if some global table setup already exists.
  */
 const getRegionsToCreateGlobalTablesIn = async function getRegionsToCreateGlobalTablesIn(
-  dynamodb, region, newRegions, tableName, cli
+  dynamodb, region, newRegions, tableName, version, cli
 ) {
   try {
-    const resp = await dynamodb.describeGlobalTable({ GlobalTableName: tableName }).promise()
-    const regionsGlobalTableExists = resp.GlobalTableDescription.ReplicationGroup.map(rg => rg.RegionName);
-    const missingRegions = [region].concat(newRegions).filter(r => !regionsGlobalTableExists.includes(r));
+    let regionsGlobalTableExists = [];
+    let missingRegions = []
+    if (version === 'v2') {
+      let resp = await dynamodb.describeTable({ TableName: tableName }).promise()
+      if (resp.Table.Replicas !== undefined) {
+        regionsGlobalTableExists = resp.Table.Replicas.map(rg => rg.RegionName);
+      }
+      missingRegions = newRegions.filter(r => !regionsGlobalTableExists.includes(r));
+    } else {
+      let resp = await dynamodb.describeGlobalTable({ GlobalTableName: tableName }).promise()
+      regionsGlobalTableExists = resp.GlobalTableDescription.ReplicationGroup.map(rg => rg.RegionName);
+      missingRegions = [region].concat(newRegions).filter(r => !regionsGlobalTableExists.includes(r));
+    }
+
     if (missingRegions.length === 0) {
       cli.consoleLog(`CreateGlobalTable: ${chalk.yellow(`Global table ${tableName} already exists in all the specified regions. Skipping creation...`)}`)
       return {missingRegions: [], addingNewRegions: false };
@@ -218,23 +230,23 @@ const getRegionsToCreateGlobalTablesIn = async function getRegionsToCreateGlobal
  * @param {string} region AWS region in which source table exists
  * @param {string} tableName Dymanodb table name
  * @param {Array} newRegions List of regions in which global table needs to be setup
+ * @param {string} version It's version of global table needs to be setup
  * @param {boolean} createStack flag indicating if the tables were created using cloudformation
  * @param {Object} cli Serverless cli object
  */
 const createGlobalTable = async function createGlobalTable(
-  appAutoScaling, dynamodb, creds, region, tableName, newRegions, createStack, cli
+  appAutoScaling, dynamodb, creds, region, tableName, newRegions, version, createStack, cli
 ) {
 
   const { missingRegions: regionsToUpdate, addingNewRegions } = await module.exports.getRegionsToCreateGlobalTablesIn(
-    dynamodb, region, newRegions, tableName, cli
+    dynamodb, region, newRegions, tableName, version, cli
   );
-
   if (!regionsToUpdate.length) {
     cli.consoleLog(`CreateGlobalTable: ${chalk.yellow(`Global table setup already in place.`)}`);
     return;
   }
 
-  if (!createStack) {
+  if (!createStack && version !== 'v2') {
     const tableDef = await dynamodb.describeTable({ TableName: tableName }).promise()
     const { ReadCapacityUnits, WriteCapacityUnits } = tableDef.Table.ProvisionedThroughput
     const { GlobalSecondaryIndexes, LocalSecondaryIndexes } = tableDef.Table;
@@ -301,6 +313,16 @@ const createGlobalTable = async function createGlobalTable(
     }));
   }
 
+  if (version === 'v2') {
+    await module.exports.createGlobalTableV2(dynamodb, tableName, regionsToUpdate, cli);
+  } else {
+    await module.exports.createGlobalTableV1(dynamodb, tableName, region, regionsToUpdate, addingNewRegions, cli);
+  }
+
+}
+
+const createGlobalTableV1 = async function createGlobalTableV1(dynamodb, tableName, region, regionsToUpdate, addingNewRegions, cli) {
+  cli.consoleLog(`CreateGlobalTable: ${chalk.yellow(`Create global table setup (Version 2017.11.29) for ${tableName}...`)}`)
   if (!addingNewRegions) {
     const replicationGroup = [{ RegionName: region }]
     regionsToUpdate.forEach(r => replicationGroup.push({ RegionName: r }))
@@ -310,7 +332,6 @@ const createGlobalTable = async function createGlobalTable(
       ReplicationGroup: replicationGroup,
     }
     await dynamodb.createGlobalTable(param).promise()
-    cli.consoleLog(`CreateGlobalTable: ${chalk.yellow(`Created global table setup for ${tableName}...`)}`)
   } else {
     const replicaUpdates = [];
     regionsToUpdate.forEach(r => replicaUpdates.push({ Create:{ RegionName: r }}));
@@ -319,8 +340,25 @@ const createGlobalTable = async function createGlobalTable(
       ReplicaUpdates: replicaUpdates,
     }
     await dynamodb.updateGlobalTable(param).promise();
-    cli.consoleLog(`CreateGlobalTable: ${chalk.yellow(`Created global table setup for ${tableName}...`)}`)
   }
+  cli.consoleLog(`CreateGlobalTable: ${chalk.yellow(`Created global table setup (Version 2017.11.29) for ${tableName}...`)}`)
+}
+
+const createGlobalTableV2 = async function createGlobalTableV2(dynamodb, tableName, regionsToUpdate, cli) {
+  cli.consoleLog(`CreateGlobalTable: ${chalk.yellow(`Create global table setup (Version 2019.11.21) for ${tableName}...`)}`)
+  for (const region of regionsToUpdate) {
+    const params = {
+      TableName: tableName,
+      ReplicaUpdates: [{ Create:{ RegionName: region }}],
+    }
+    cli.consoleLog(`CreateGlobalTable: ${chalk.yellow(`Wait for ${tableName} replication available...`)}`)
+    await dynamodb.waitFor('tableExists', {TableName: tableName}).promise(); // it's gonna wait for "Active" status
+    cli.consoleLog(`CreateGlobalTable: ${chalk.yellow(`Start creating a replica for ${tableName} in ${region}`)}`)
+    await dynamodb.updateTable(params).promise();
+    await dynamodb.waitFor('tableExists', {TableName: tableName}).promise(); // it's gonna wait for "Active" status
+    cli.consoleLog(`CreateGlobalTable: ${chalk.yellow(`The replica for ${tableName} in ${region} has been created successfully`)}`)
+  }
+  cli.consoleLog(`CreateGlobalTable: ${chalk.yellow(`The global table setup (Version 2019.11.21) for ${tableName} has been created successfully`)}`)
 }
 
 /**
@@ -392,6 +430,7 @@ const createGlobalDynamodbTable = async function createGlobalDynamodbTable(serve
           region,
           tableName,
           globalTablesOptions.regions,
+          globalTablesOptions.version,
           false,
           cli
         )
@@ -414,6 +453,7 @@ const createGlobalDynamodbTable = async function createGlobalDynamodbTable(serve
           region,
           tableName,
           globalTablesOptions.regions,
+          globalTablesOptions.version,
           true,
           cli
         )
@@ -428,6 +468,8 @@ module.exports = {
   checkStackCreateUpdateStatus,
   createGlobalDynamodbTable,
   createGlobalTable,
+  createGlobalTableV1,
+  createGlobalTableV2,
   createNewTableAndSetScalingPolicy,
   createUpdateCfnStack,
   getRegionsToCreateGlobalTablesIn,

src/helper.js

@@ -414,7 +414,7 @@ describe('test getRegionsToCreateGlobalTablesIn function', () =>{
     'us-west-1',
     'us-east-1'
   ]
-  describe('no global table exists', () => {
+  describe('no global table exists for v1', () => {
     before(() => {
       const error = new Error('forced error');
       error.code = 'GlobalTableNotFoundException';
@@ -426,7 +426,26 @@ describe('test getRegionsToCreateGlobalTablesIn function', () =>{
       dynamodb.describeGlobalTable.restore();
     });
     it ('should return all the new regions', async () => {
-      const resp = await plugin.getRegionsToCreateGlobalTablesIn(dynamodb, 'us-west-2', newRegions, 'test-table', serverless.cli);
+      const resp = await plugin.getRegionsToCreateGlobalTablesIn(dynamodb, 'us-west-2', newRegions, 'test-table', 'v1', serverless.cli);
+      resp.missingRegions.should.have.length(2);
+      resp.missingRegions.should.eql(newRegions);
+      resp.addingNewRegions.should.eql(false);
+    });
+  });
+
+  describe('no global table exists for v2', () => {
+    before(() => {
+      const error = new Error('forced error');
+      error.code = 'GlobalTableNotFoundException';
+      sinon.stub(dynamodb, 'describeTable').returns({
+        promise: () => { return Promise.reject(error)}
+      });
+    });
+    after(() => {
+      dynamodb.describeTable.restore();
+    });
+    it ('should return all the new regions', async () => {
+      const resp = await plugin.getRegionsToCreateGlobalTablesIn(dynamodb, 'us-west-2', newRegions, 'test-table', 'v2', serverless.cli);
       resp.missingRegions.should.have.length(2);
       resp.missingRegions.should.eql(newRegions);
       resp.addingNewRegions.should.eql(false);
@@ -446,7 +465,7 @@ describe('test getRegionsToCreateGlobalTablesIn function', () =>{
     });
     it ('should return all the new regions', async () => {
       try {
-        await plugin.getRegionsToCreateGlobalTablesIn(dynamodb, 'us-west-2', newRegions, 'test-table', serverless.cli);
+        await plugin.getRegionsToCreateGlobalTablesIn(dynamodb, 'us-west-2', newRegions, 'test-table', 'v1', serverless.cli);
       } catch (err) {
         err.code.should.eql('RandomError');
       }
@@ -471,7 +490,7 @@ describe('test getRegionsToCreateGlobalTablesIn function', () =>{
       dynamodb.describeGlobalTable.restore();
     });
     it ('should return no region', async () => {
-      const resp = await plugin.getRegionsToCreateGlobalTablesIn(dynamodb, 'us-west-2', newRegions, 'test-table', serverless.cli);
+      const resp = await plugin.getRegionsToCreateGlobalTablesIn(dynamodb, 'us-west-2', newRegions, 'test-table', 'v1', serverless.cli);
       resp.missingRegions.should.have.length(0);
       resp.addingNewRegions.should.eql(false);
     });
@@ -494,7 +513,31 @@ describe('test getRegionsToCreateGlobalTablesIn function', () =>{
       dynamodb.describeGlobalTable.restore();
     });
     it ('should return no region', async () => {
-      const resp = await plugin.getRegionsToCreateGlobalTablesIn(dynamodb, 'us-west-2', newRegions, 'test-table', serverless.cli);
+      const resp = await plugin.getRegionsToCreateGlobalTablesIn(dynamodb, 'us-west-2', newRegions, 'test-table', 'v1', serverless.cli);
+      resp.missingRegions.should.have.length(1);
+      resp.missingRegions[0].should.eql('us-east-1');
+      resp.addingNewRegions.should.eql(true);
+    });
+  });
+
+  describe('global table exists in all the specified new regions for v2', () => {
+    before(() => {
+      sinon.stub(dynamodb, 'describeTable').returns({
+        promise: () => { return Promise.resolve({
+          Table: {
+            Replicas: [
+              { RegionName: 'us-west-1'},
+              { RegionName: 'us-west-2'}
+            ]
+          }
+        })}
+      });
+    });
+    after(() => {
+      dynamodb.describeTable.restore();
+    });
+    it ('should return no region', async () => {
+      const resp = await plugin.getRegionsToCreateGlobalTablesIn(dynamodb, 'us-west-2', newRegions, 'test-table', 'v2', serverless.cli);
       resp.missingRegions.should.have.length(1);
       resp.missingRegions[0].should.eql('us-east-1');
       resp.addingNewRegions.should.eql(true);
@@ -557,6 +600,12 @@ describe('test createGlobalTable function', () => {
     sandbox.stub(dynamodb, 'updateGlobalTable').returns({
       promise: () => { return Promise.resolve()}
     });
+    sandbox.stub(dynamodb, 'updateTable').returns({
+      promise: () => { return Promise.resolve()}
+    });
+    sandbox.stub(dynamodb, 'waitFor').returns({
+      promise: () => { return Promise.resolve()}
+    });
   });
   afterEach(() => {
     sandbox.restore();
@@ -568,12 +617,12 @@ describe('test createGlobalTable function', () => {
       missingRegions: [],
       addingNewRegions: false
     }));
-    await plugin.createGlobalTable(aas, dynamodb, serverless.getProvider().getCredentials(), 'us-west-2', 'test-table', ['us-east-2'], true, serverless.cli);
+    await plugin.createGlobalTable(aas, dynamodb, serverless.getProvider().getCredentials(), 'us-west-2', 'test-table', ['us-east-2'], 'v1',true, serverless.cli);
     sandbox.assert.notCalled(dynamodb.createGlobalTable);
   });
 
   it ('should create global table', async () => {
-    await plugin.createGlobalTable(aas, dynamodb, serverless.getProvider().getCredentials(), 'us-west-2', 'test-table', ['us-east-2'], true, serverless.cli);
+    await plugin.createGlobalTable(aas, dynamodb, serverless.getProvider().getCredentials(), 'us-west-2', 'test-table', ['us-east-2'], 'v1', true, serverless.cli);
     sandbox.assert.calledOnce(dynamodb.createGlobalTable);
   });
 
@@ -583,14 +632,14 @@ describe('test createGlobalTable function', () => {
       missingRegions: ['us-west-1'],
       addingNewRegions: true
     }));
-    await plugin.createGlobalTable(aas, dynamodb, serverless.getProvider().getCredentials(), 'us-west-2', 'test-table', ['us-east-2'], true, serverless.cli);
+    await plugin.createGlobalTable(aas, dynamodb, serverless.getProvider().getCredentials(), 'us-west-2', 'test-table', ['us-east-2'], 'v1', true, serverless.cli);
     sandbox.assert.notCalled(dynamodb.createGlobalTable);
     sandbox.assert.calledOnce(dynamodb.updateGlobalTable);
   });
 
   context("when create stack is false", () => {
     it ('should create the table with ProvisionedThroughput if billing mode is not PAY_PER_REQUEST', async () => {
-      await plugin.createGlobalTable(aas, dynamodb, serverless.getProvider().getCredentials(), 'us-west-2', 'test-table', ['us-east-2'], false, serverless.cli);
+      await plugin.createGlobalTable(aas, dynamodb, serverless.getProvider().getCredentials(), 'us-west-2', 'test-table', ['us-east-2'], 'v1', false, serverless.cli);
       sandbox.assert.calledOnce(dynamodb.describeTable);
       plugin.createNewTableAndSetScalingPolicy.lastCall.args[2].should.eql({ AttributeDefinitions: {},
         KeySchema: '',
@@ -612,7 +661,7 @@ describe('test createGlobalTable function', () => {
 
     it ('should create the table without ProvisionedThroughput if billing mode is PAY_PER_REQUEST', async () => {
       stubbedTable.Table.BillingModeSummary = { BillingMode: "PAY_PER_REQUEST" };
-      await plugin.createGlobalTable(aas, dynamodb, serverless.getProvider().getCredentials(), 'us-west-2', 'test-table', ['us-east-2'], false, serverless.cli);
+      await plugin.createGlobalTable(aas, dynamodb, serverless.getProvider().getCredentials(), 'us-west-2', 'test-table', ['us-east-2'], 'v1', false, serverless.cli);
       sandbox.assert.calledOnce(dynamodb.describeTable);
       plugin.createNewTableAndSetScalingPolicy.lastCall.args[2].should.eql({ AttributeDefinitions: {},
         KeySchema: '',
@@ -628,6 +677,13 @@ describe('test createGlobalTable function', () => {
         BillingMode: 'PAY_PER_REQUEST',
         Tags: {} });
     });
+
+    it ('should create the table by using v2 version', async () => {
+      stubbedTable.Table.BillingModeSummary = { BillingMode: "PAY_PER_REQUEST" };
+      await plugin.createGlobalTable(aas, dynamodb, serverless.getProvider().getCredentials(), 'us-west-2', 'test-table', ['us-east-2'], 'v2', false, serverless.cli);
+      sandbox.assert.calledOnce(dynamodb.updateTable);
+      sandbox.assert.calledTwice(dynamodb.waitFor);
+    });
   })
 });