DynamoDB Local: Streams.getRecords returns TrimmedDataAccessException

Question

I'm using DynamoDB local to run integration tests from nodejs (Javascript SDK). When I call getRecords I'm getting "TrimmedDataAccessException". If I run the same code against DynamoDB in AWS (not local) it works fine. Here's the steps:

• Run DynamoDB
• createTable "EventStore"
• createTable "Music"
• describeTable "EventStore" (gets the LatestStreamArn")
• describeStream returned as "LatestStreamArn" (gets the ShardId)
• getShardIterator with ShardIteratorType "LATEST"
• put record to "Music" table
• put record to "EventStore" table
• getRecords from "EventStore" stream

The order of the "put" commands is important. If I put to "EventStore" first everything works fine, but when I put to "Music" first it fails. My application logic would be wrong if I change the order though so just changing the order is a problem for me.

Run DynamoDB

java -Djava.library.path=./DynamoDBLocal_lib -jar DynamoDBLocal.jar -sharedDb -dbPath /data

createTable

{
    TableName: "EventStore",
    KeySchema: [
        { AttributeName: "EntityId", KeyType: "HASH" },
        { AttributeName: "Version", KeyType: "RANGE" }
    ],
    AttributeDefinitions: [
        { AttributeName: "EntityId", AttributeType: "S" },
        { AttributeName: "Version", AttributeType: "N" }
    ],
    ProvisionedThroughput: {
        ReadCapacityUnits: 10,
        WriteCapacityUnits: 10
    },
    StreamSpecification: {
        StreamEnabled: true,
        StreamViewType: "NEW_IMAGE"
    }
}

createTable (response)

{
  "TableDescription": {
    "AttributeDefinitions": [
      {
        "AttributeName": "EntityId",
        "AttributeType": "S"
      },
      {
        "AttributeName": "Version",
        "AttributeType": "N"
      }
    ],
    "TableName": "EventStore",
    "KeySchema": [
      {
        "AttributeName": "EntityId",
        "KeyType": "HASH"
      },
      {
        "AttributeName": "Version",
        "KeyType": "RANGE"
      }
    ],
    "TableStatus": "ACTIVE",
    "CreationDateTime": "2016-07-14T15:36:42.895Z",
    "ProvisionedThroughput": {
      "LastIncreaseDateTime": "1970-01-01T00:00:00.000Z",
      "LastDecreaseDateTime": "1970-01-01T00:00:00.000Z",
      "NumberOfDecreasesToday": 0,
      "ReadCapacityUnits": 10,
      "WriteCapacityUnits": 10
    },
    "TableSizeBytes": 0,
    "ItemCount": 0,
    "TableArn": "arn:aws:dynamodb:ddblocal:000000000000:table/EventStore",
    "StreamSpecification": {
      "StreamEnabled": true,
      "StreamViewType": "NEW_IMAGE"
    },
    "LatestStreamLabel": "2016-07-14T15:36:42.895",
    "LatestStreamArn": "arn:aws:dynamodb:ddblocal:000000000000:table/EventStore/stream/2016-07-14T15:36:42.895"
}

describeTable(EventStore)

{
    "TableName": "EventStore"
}

describeTable(EventStore) response

{
  "Table": {
    "AttributeDefinitions": [
      {
        "AttributeName": "EntityId",
        "AttributeType": "S"
      },
      {
        "AttributeName": "Version",
        "AttributeType": "N"
      }
    ],
    "TableName": "EventStore",
    "KeySchema": [
      {
        "AttributeName": "EntityId",
        "KeyType": "HASH"
      },
      {
        "AttributeName": "Version",
        "KeyType": "RANGE"
      }
    ],
    "TableStatus": "ACTIVE",
    "CreationDateTime": "2016-07-14T15:36:42.895Z",
    "ProvisionedThroughput": {
      "LastIncreaseDateTime": "1970-01-01T00:00:00.000Z",
      "LastDecreaseDateTime": "1970-01-01T00:00:00.000Z",
      "NumberOfDecreasesToday": 0,
      "ReadCapacityUnits": 10,
      "WriteCapacityUnits": 10
    },
    "TableSizeBytes": 0,
    "ItemCount": 0,
    "TableArn": "arn:aws:dynamodb:ddblocal:000000000000:table/EventStore",
    "StreamSpecification": {
      "StreamEnabled": true,
      "StreamViewType": "NEW_IMAGE"
    },
    "LatestStreamLabel": "2016-07-14T15:36:42.895",
    "LatestStreamArn": "arn:aws:dynamodb:ddblocal:000000000000:table/EventStore/stream/2016-07-14T15:36:42.895"
  }
}

describeSteam

{ 
    "StreamArn": "arn:aws:dynamodb:ddblocal:000000000000:table/EventStore/stream/2016-07-14T15:36:42.895" 
}

describeStream response

{
  "StreamDescription": {
    "StreamArn": "arn:aws:dynamodb:ddblocal:000000000000:table/EventStore/stream/2016-07-14T15:36:42.895",
    "StreamLabel": "2016-07-14T15:36:42.895",
    "StreamStatus": "ENABLED",
    "StreamViewType": "NEW_IMAGE",
    "CreationRequestDateTime": "2016-07-14T15:36:42.895Z",
    "TableName": "EventStore",
    "KeySchema": [
      {
        "AttributeName": "EntityId",
        "KeyType": "HASH"
      },
      {
        "AttributeName": "Version",
        "KeyType": "RANGE"
      }
    ],
    "Shards": [
      {
        "ShardId": "shardId-00000001468510602897-249806fa",
        "SequenceNumberRange": {
          "StartingSequenceNumber": "000000000000000000052"
        }
      }
    ]
  }
}

getShardIterator

{
  "ShardId": "shardId-00000001468510602897-249806fa",
  "ShardIteratorType": "LATEST",
  "StreamArn": "arn:aws:dynamodb:ddblocal:000000000000:table/EventStore/stream/2016-07-14T15:36:42.895"
}

getShardIterator response

{
  "ShardIterator": "000|arn:aws:dynamodb:ddblocal:000000000000:table/EventStore/stream/2016-07-14T15:36:42.895|c2hhcmRJZC0wMDAwMDAwMTQ2ODUxMDYwMjg5Ny0yNDk4MDZmYXwwMDAwMDAwMDAwMDAwMDAwMDAwNTJ8MDAwMDAwMDAwMDAwMDAwMDAxNDY4NTEwNjc2NjEy"
}

put(TableName: Music)

{
    TableName: "Music",
    Item: {
            Id: "000-000-000-000-000",
            Artist: "No One You Know",
            SongTitle: "Call Me Today"
        },
    "ConditionExpression": "attribute_not_exists(Artist) and attribute_not_exists(SongTitle)"
}

put(TableName: EventStore)

{
  "TableName": "EventStore",
  "Item": {
    "EntityId": "000-000-000-000-000",
    "Version": 1468510676704,
    "Payload": [
      {
        "type": "Buffer",
        "data": [
            38,48,48,48,45,48,48,48,45,48,48,48,45,48,48,48,45,48,48,48,30,78,111,32,79,110,101,32,89,111,117,32,75,110,111,119,26,67,97,108,108,32,77,101,32,84,111,100,97,121
        ]
      }
    ],
    "Fingerprint": "8871e2afc3c31edfa9938e4cbb2b5",
    "Timestamp": 1468510676704
  },
  "ConditionExpression": "attribute_not_exists(EntityId) and attribute_not_exists(Version)"
}

getRecords (EventStore)

{ ShardIterator: "000|arn:aws:dynamodb:ddblocal:000000000000:table/EventStore/stream/2016-07-14T15:36:42.895|c2hhcmRJZC0wMDAwMDAwMTQ2ODUxMDYwMjg5Ny0yNDk4MDZmYXwwMDAwMDAwMDAwMDAwMDAwMDAwNTJ8MDAwMDAwMDAwMDAwMDAwMDAxNDY4NTEwNjc2NjEy" }

getRecords (EventStore) response

{
  "message": "The operation attempted to read past the oldest stream record in a shard.",
  "code": "TrimmedDataAccessException",
  "time": "2016-07-14T15:37:56.740Z",
  "requestId": "0e4d43ae-ac7d-419b-a1db-c8e2a955a877",
  "statusCode": 400,
  "retryable": false,
  "retryDelay": 30.214022053405643
}

Answer 1

I've found that if after the getRecords call, you call getShardIterator again (with the same params), it works.

Clearly a bug in the local implementation.