Merge pull request #4 from amineo/master

Add CloudFront stream to Kinesis log format

Author: claygregory

README.md

@@ -55,11 +55,11 @@ fs.createReadStream('./somelogfile.gz')
 
 ### Options
 
-Only two configuration options are currently in effect: format and version. The parser defaults to `web` to handle the web distribution file format. If logs are from an RTMP distribution, this value should be set to `rtmp`. Currently all CloudFront logs are on version 1.0; should future versions appear, the `version` option will serve as an override.
+Only two configuration options are currently in effect: format and version. The parser defaults to `web` to handle the web distribution file format. If logs are streaming from CloudFront to Kinesis the format should be set to `kinesis` as the column headers and order changes. If logs are from an RTMP distribution, this value should be set to `rtmp`. Currently all CloudFront logs are on version 1.0; should future versions appear, the `version` option will serve as an override.
 
 ```javascript
 const options = {
-  format: 'web|rtmp',
+  format: 'web|rtmp|kinesis',
   version: '1.0'
 };
 ```
@@ -95,6 +95,51 @@ const options = {
   'cs-protocol-version': 'HTTP/1.1' }
 ```
 
+
+### CloudFront to Kinesis Distribution Format
+```javascript
+{ 'timestamp': '1607374321.541',
+  'c-ip': '127.0.0.1',
+  'time-to-first-byte': '0.042',
+  'sc-status': '200',
+  'sc-bytes': '485',
+  'cs-method': 'GET',
+  'cs-protocol': 'http',
+  'cs-host': 'test.cloudfront.net',
+  'cs-uri-stem': '/i?hello=1',
+  'cs-bytes': '745',
+  'x-edge-location': 'EWR52-C4',
+  'x-edge-request-id': '6PfZe0cc_AjXUjFuGnL9pGOmFdUx8xR8ZU8nr44JYJWi-DaeJjcxkw==',
+  'x-host-header': 'test.cloudfront.net',
+  'time-taken': '0.042',
+  'cs-protocol-version': 'HTTP/1.1',
+  'c-ip-version': 'IPv4',
+  'cs-user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:83.0) Gecko/20100101 Firefox/83.0',
+  'cs-referer': 'http://localhost:5000/page-2',
+  'cs-cookie': '-',
+  'query-params': 'hello=1607374321563',
+  'x-edge-response-result-type': 'Miss',
+  'x-forwarded-for': '-',
+  'ssl-protocol': '-',
+  'ssl-cipher': '-',
+  'x-edge-result-type': 'Miss',
+  'fle-encrypted-fields': '-',
+  'fle-status': '-',
+  'sc-content-type': 'image/gif',
+  'sc-content-len': '35',
+  'sc-range-start': '-',
+  'sc-range-end': '-',
+  'c-port': '49323',
+  'x-edge-detailed-result-type': 'Miss',
+  'c-country': 'US',
+  'cs-accept-encoding': 'gzip, deflate',
+  'cs-accept': 'image/webp,*/*',
+  'cache-behavior-path-pattern': '*',
+  'cs-headers': 'Host:test.cloudfront.net\n...',
+  'cs-headers-count': '8'
+}
+```
+
 ### RTMP Distribution Format
 
 ```javascript

index.js

@@ -22,7 +22,21 @@ const formats = {
     'sc-bytes', 'x-cf-status', 'x-cf-client-id', 'cs-uri-stem',
     'cs-uri-query', 'c-referrer', 'x-page-url', 'c-user-agent',
     'x-sname', 'x-sname-query', 'x-file-ext', 'x-sid'
+  ],
+
+  'kinesis_v1.0': [
+    'timestamp', 'c-ip', 'time-to-first-byte', 'sc-status',
+    'sc-bytes', 'cs-method', 'cs-protocol', 'cs-host',
+    'cs-uri-stem', 'cs-bytes', 'x-edge-location', 'x-edge-request-id',
+    'x-host-header', 'time-taken', 'cs-protocol-version', 'c-ip-version',
+    'cs-user-agent', 'cs-referer', 'cs-cookie', 'cs-uri-query',
+    'x-edge-response-result-type', 'x-forwarded-for', 'ssl-protocol', 'ssl-cipher',
+    'x-edge-result-type', 'fle-encrypted-fields', 'fle-status', 'sc-content-type',
+    'sc-content-len', 'sc-range-start', 'sc-range-end', 'c-port',
+    'x-edge-detailed-result-type', 'c-country', 'cs-accept-encoding', 'cs-accept',
+    'cache-behavior-path-pattern', 'cs-headers', 'cs-header-names', 'cs-headers-count'
   ]
+
 };
 
 const option_defaults = {

package-lock.json

@@ -0,0 +1,3 @@
+#Version: 1.0
+#Fields: timestamp c-ip time-to-first-byte sc-status sc-bytes cs-method cs-protocol cs-host cs-uri-stem cs-bytes x-edge-location x-edge-request-id x-host-header time-taken cs-protocol-version c-ip-version cs-user-agent cs-referer cs-cookie cs-uri-query x-edge-response-result-type x-forwarded-for ssl-protocol ssl-cipher x-edge-result-type fle-encrypted-fields fle-status sc-content-type sc-content-len sc-range-start sc-range-end c-port x-edge-detailed-result-type c-country cs-accept-encoding cs-accept cache-behavior-path-pattern cs-headers cs-header-names cs-headers-count
+1607374321.541	127.0.0.1	0.042	200	485	GET	http	test.cloudfront.net	/i?stm=1607374321563&e=pp&url=http%253A%252F%252Flocalhost%253A5000%252Fpage-2&refr=http%253A%252F%252Flocalhost%253A5000%252F&pp_mix=0&pp_max=0&pp_miy=0&pp_may=0&tv=js-2.6.2&tna=cf&aid=site&p=web&tz=America%252FNew_York&lang=en-US&cs=UTF-8&res=3840x1600&cd=24&cookie=1&eid=d778127d-4ddf-4c04-90af-ffccc980eee8&dtm=1607374321561&vp=2450x1431&ds=2450x1431&vid=5&sid=a88ec782-713b-4e0d-b2da-c408e5730834&duid=eaa664f5-8ba9-41e9-9c98-a2ad088ca440&fp=2033130908	745	EWR52-C4	6PfZe0cc_AjXUjFuGnL9pGOmFdUx8xR8ZU8nr44JYJWi-DaeJjcxkw==	test.cloudfront.net	0.042	HTTP/1.1	IPv4	Mozilla/5.0%20(Macintosh;%20Intel%20Mac%20OS%20X%2010.15;%20rv:83.0)%20Gecko/20100101%20Firefox/83.0	http://localhost:5000/page-2	-	stm=1607374321563&e=pp&url=http%253A%252F%252Flocalhost%253A5000%252Fpage-2&refr=http%253A%252F%252Flocalhost%253A5000%252F&pp_mix=0&pp_max=0&pp_miy=0&pp_may=0&tv=js-2.6.2&tna=cf&aid=site&p=web&tz=America%252FNew_York&lang=en-US&cs=UTF-8&res=3840x1600&cd=24&cookie=1&eid=d778127d-4ddf-4c04-90af-ffccc980eee8&dtm=1607374321561&vp=2450x1431&ds=2450x1431&vid=5&sid=a88ec782-713b-4e0d-b2da-c408e5730834&duid=eaa664f5-8ba9-41e9-9c98-a2ad088ca440&fp=2033130908	Miss	-	-	-	Miss	-	-	image/gif	35	-	-	49323	Miss	US	gzip,%20deflate	image/webp,*/*	*	Host:test.cloudfront.net%0AUser-Agent:Mozilla/5.0%20(Macintosh;%20Intel%20Mac%20OS%20X%2010.15;%20rv:83.0)%20Gecko/20100101%20Firefox/83.0%0AAccept:image/webp,*/*%0AAccept-Language:en-US,en;q=0.5%0AAccept-Encoding:gzip,%20deflate%0ADNT:1%0AConnection:keep-alive%0AReferer:http://localhost:5000/page-2%0A	Host%0AUser-Agent%0AAccept%0AAccept-Language%0AAccept-Encoding%0ADNT%0AConnection%0AReferer%0A	8

test/fixtures/kinesis-1-0.txt

@@ -11,6 +11,7 @@ const CloudFrontParser = require('../');
 const web_example = fs.readFileSync(path.join(__dirname, './fixtures/web-1-0.txt'), 'utf-8');
 const web_example1 = fs.readFileSync(path.join(__dirname, './fixtures/web-1-0-2019-12.txt'), 'utf-8');
 const rtmp_example = fs.readFileSync(path.join(__dirname, './fixtures/rtmp-1-0.txt'), 'utf-8');
+const kinesis_example = fs.readFileSync(path.join(__dirname, './fixtures/kinesis-1-0.txt'), 'utf-8');
 
 describe('parse', function () {
 
@@ -26,6 +27,11 @@ describe('parse', function () {
     CloudFrontParser.parse(rtmp_example, { format: 'rtmp' });
   });
 
+  it('should parse Kinesis to Cloudfront v1.0 logs without error', function () {
+    CloudFrontParser.parse(kinesis_example, { format: 'kinesis' });
+  });
+
+
   it('should create a single object out of each line of web log, ignoring comments', function () {
     const result = CloudFrontParser.parse(web_example, { format: 'web' });
     assert.equal(2, result.length);
@@ -36,6 +42,12 @@ describe('parse', function () {
     assert.equal(6, result.length);
   });
 
+
+  it('should create a single object out of each line of kinesis log, ignoring comments', function () {
+    const result = CloudFrontParser.parse(kinesis_example, { format: 'kinesis' });
+    assert.equal(1, result.length);
+  });
+
   it('should default to web if format unspecified', function () {
     const result = CloudFrontParser.parse(web_example);
 
@@ -77,6 +89,19 @@ describe('parse', function () {
     assert.equal('disconnect', result[5]['x-event']);
   });
 
+
+  it('should map each kinesis log field into correct result field', function () {
+    const result = CloudFrontParser.parse(kinesis_example, { format: 'kinesis' });
+
+    assert.equal('1607374321.541', result[0]['timestamp']);
+    assert.equal('127.0.0.1', result[0]['c-ip']);
+    assert.equal('0.042', result[0]['time-to-first-byte']);
+    assert.equal('200', result[0]['sc-status']);
+    assert.equal('Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:83.0) Gecko/20100101 Firefox/83.0', result[0]['cs-user-agent']);
+  });
+
+
+
   it('should correctly decode percent-encoded fields', function () {
     const result = CloudFrontParser.parse(web_example, { format: 'web' });
     assert.equal('Mozilla/5.0 (iPhone; CPU iPhone OS 10_2_1 like Mac OS X) AppleWebKit/602.4.6 (KHTML, like Gecko) Version/10.0 Mobile/14D27 Safari/602.1', result[0]['cs-user-agent']);