Skip to content

Commit 2ad9d81

Browse files
ricardo-devis-agulloricardo-devis-agullo
authored
Merge pull request #1319 from opencomponents/check-packages
Check unverified packages after start/publishing
2 parents bcc73f2 + c974a9d commit 2ad9d81

5 files changed

Lines changed: 123 additions & 21 deletions

File tree

‎.npmignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,3 +7,4 @@ appveyor.yml
77
Gruntfile.js
88
tasks
99
test
10+
src

‎src/registry/domain/components-cache/components-list.ts‎

Lines changed: 54 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
import semver from 'semver';
22
import pLimit from 'p-limit';
33
import getUnixUTCTimestamp from 'oc-get-unix-utc-timestamp';
4-
import { ComponentsList, Config } from '../../../types';
54
import { StorageAdapter } from 'oc-storage-adapters-utils';
5+
import eventsHandler from '../events-handler';
6+
import { ComponentsList, Config } from '../../../types';
67

78
export default function componentsList(conf: Config, cdn: StorageAdapter) {
89
const filePath = (): string =>
@@ -11,17 +12,64 @@ export default function componentsList(conf: Config, cdn: StorageAdapter) {
1112
const componentsList = {
1213
getFromJson: (): Promise<ComponentsList> => cdn.getJson(filePath(), true),
1314

14-
getFromDirectories: async (): Promise<ComponentsList> => {
15+
getFromDirectories: async (
16+
jsonList: ComponentsList | null
17+
): Promise<ComponentsList> => {
1518
const componentsInfo: Record<string, string[]> = {};
1619

20+
const validateComponentVersion = (
21+
componentName: string,
22+
componentVersion: string
23+
) => {
24+
return cdn
25+
.getJson(
26+
// Check integrity of the package by checking existence of package.json
27+
// OC will upload always the package.json last when publishing
28+
`${conf.storage.options.componentsDir}/${componentName}/${componentVersion}/package.json`
29+
)
30+
.then(() => true)
31+
.catch(() => false);
32+
};
33+
1734
const getVersionsForComponent = async (
1835
componentName: string
1936
): Promise<string[]> => {
20-
const versions = await cdn.listSubDirectories(
37+
const allVersions = await cdn.listSubDirectories(
2138
`${conf.storage.options.componentsDir}/${componentName}`
2239
);
40+
const unCheckedVersions = allVersions.filter(
41+
version => !jsonList?.components[componentName]?.includes(version)
42+
);
43+
const limit = pLimit(cdn.maxConcurrentRequests);
44+
const invalidVersions = (
45+
await Promise.all(
46+
unCheckedVersions.map(unCheckedVersion =>
47+
limit(async () => {
48+
const isValid = await validateComponentVersion(
49+
componentName,
50+
unCheckedVersion
51+
);
52+
53+
return isValid ? null : unCheckedVersion;
54+
})
55+
)
56+
)
57+
).filter((x): x is string => typeof x === 'string');
58+
59+
if (invalidVersions.length > 0) {
60+
eventsHandler.fire('error', {
61+
code: 'corrupted_version',
62+
message: `Couldn't validate the integrity of the component ${componentName} on the following versions: ${invalidVersions.join(
63+
', '
64+
)}.`
65+
});
66+
}
67+
68+
const validVersions = allVersions.filter(
69+
version => !invalidVersions.includes(version)
70+
);
2371

24-
return versions.sort(semver.compare);
72+
return validVersions.sort(semver.compare);
2573
};
2674

2775
try {
@@ -55,8 +103,8 @@ export default function componentsList(conf: Config, cdn: StorageAdapter) {
55103
}
56104
},
57105

58-
async refresh(): Promise<ComponentsList> {
59-
const components = await componentsList.getFromDirectories();
106+
async refresh(cachedList: ComponentsList): Promise<ComponentsList> {
107+
const components = await componentsList.getFromDirectories(cachedList);
60108
await componentsList.save(components);
61109

62110
return components;

‎src/registry/domain/components-cache/index.ts‎

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -3,16 +3,16 @@ import getComponentsList from './components-list';
33
import eventsHandler from '../events-handler';
44
import getUnixUTCTimestamp from 'oc-get-unix-utc-timestamp';
55
import { ComponentsList, Config } from '../../../types';
6-
import { StorageAdapter } from 'oc-storage-adapters-utils';
6+
import { StorageAdapter, strings } from 'oc-storage-adapters-utils';
77

88
export default function componentsCache(conf: Config, cdn: StorageAdapter) {
99
let cachedComponentsList: ComponentsList;
1010
let refreshLoop: NodeJS.Timeout;
1111

1212
const componentsList = getComponentsList(conf, cdn);
1313

14-
const poll = () =>
15-
setTimeout(async () => {
14+
const poll = () => {
15+
return setTimeout(async () => {
1616
try {
1717
const data = await componentsList.getFromJson();
1818

@@ -29,6 +29,7 @@ export default function componentsCache(conf: Config, cdn: StorageAdapter) {
2929
}
3030
refreshLoop = poll();
3131
}, conf.pollingInterval * 1000);
32+
};
3233

3334
const cacheDataAndStartPolling = (data: ComponentsList) => {
3435
cachedComponentsList = data;
@@ -55,12 +56,15 @@ export default function componentsCache(conf: Config, cdn: StorageAdapter) {
5556
},
5657

5758
async load(): Promise<ComponentsList> {
59+
const jsonComponents = await componentsList.getFromJson().catch(err => {
60+
if (err?.code === strings.errors.STORAGE.FILE_NOT_FOUND_CODE)
61+
return null;
62+
63+
return Promise.reject(err);
64+
});
5865
const dirComponents = await componentsList
59-
.getFromDirectories()
66+
.getFromDirectories(jsonComponents)
6067
.catch(err => throwError('components_list_get', err));
61-
const jsonComponents = await componentsList
62-
.getFromJson()
63-
.catch(() => null);
6468

6569
if (
6670
!jsonComponents ||
@@ -78,7 +82,8 @@ export default function componentsCache(conf: Config, cdn: StorageAdapter) {
7882
async refresh(): Promise<ComponentsList> {
7983
clearTimeout(refreshLoop);
8084
try {
81-
const components = await componentsList.refresh();
85+
// Passing components that we know are fine, so it doesn't refresh invalid components
86+
const components = await componentsList.refresh(cachedComponentsList);
8287
cacheDataAndStartPolling(components);
8388

8489
return components;

‎test/unit/registry-domain-components-cache.js‎

Lines changed: 54 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,8 @@ describe('registry : domain : components-cache', () => {
4545
'./components-list': injectr(
4646
'../../dist/registry/domain/components-cache/components-list.js',
4747
{
48-
'oc-get-unix-utc-timestamp': getTimestamp
48+
'oc-get-unix-utc-timestamp': getTimestamp,
49+
'../events-handler': eventsHandlerStub
4950
}
5051
).default
5152
},
@@ -59,10 +60,27 @@ describe('registry : domain : components-cache', () => {
5960
};
6061

6162
describe('when library does not contain components.json', () => {
63+
describe('when getting the json fails', () => {
64+
let error;
65+
before(done => {
66+
mockedCdn.getJson = sinon.stub();
67+
mockedCdn.getJson.rejects(new Error('FILE_ERROR'));
68+
initialise();
69+
componentsCache
70+
.load()
71+
.catch(err => (error = err))
72+
.finally(done);
73+
});
74+
75+
it('should throw with the error message', () => {
76+
expect(error.message).to.equal('FILE_ERROR');
77+
});
78+
});
6279
describe('when initialising the cache', () => {
6380
before(done => {
6481
mockedCdn.getJson = sinon.stub();
65-
mockedCdn.getJson.rejects('not_found');
82+
mockedCdn.getJson.resolves({});
83+
mockedCdn.getJson.onFirstCall(0).rejects({ code: 'file_not_found' });
6684
mockedCdn.listSubDirectories = sinon.stub();
6785
mockedCdn.listSubDirectories.onCall(0).resolves(['hello-world']);
6886
mockedCdn.listSubDirectories.onCall(1).resolves(['1.0.0', '1.0.2']);
@@ -72,11 +90,17 @@ describe('registry : domain : components-cache', () => {
7290
componentsCache.load().finally(done);
7391
});
7492

75-
it('should try fetching the components.json', () => {
76-
expect(mockedCdn.getJson.calledOnce).to.be.true;
93+
it('should try fetching the components.json and check components', () => {
94+
expect(mockedCdn.getJson.calledThrice).to.be.true;
7795
expect(mockedCdn.getJson.args[0][0]).to.be.equal(
7896
'component/components.json'
7997
);
98+
expect(mockedCdn.getJson.args[1][0]).to.be.equal(
99+
'component/hello-world/1.0.0/package.json'
100+
);
101+
expect(mockedCdn.getJson.args[2][0]).to.be.equal(
102+
'component/hello-world/1.0.2/package.json'
103+
);
80104
});
81105

82106
it('should scan for directories to fetch components and versions', () => {
@@ -106,24 +130,48 @@ describe('registry : domain : components-cache', () => {
106130
before(done => {
107131
mockedCdn.getJson = sinon.stub();
108132
mockedCdn.getJson.resolves(baseResponse());
133+
mockedCdn.getJson
134+
.withArgs('component/hello-world/3.0.0/package.json')
135+
.rejects('ERROR');
109136
mockedCdn.listSubDirectories = sinon.stub();
110137
mockedCdn.listSubDirectories.onCall(0).resolves(['hello-world']);
111138
mockedCdn.listSubDirectories
112139
.onCall(1)
113-
.resolves(['1.0.0', '1.0.2', '2.0.0']);
140+
.resolves(['1.0.0', '1.0.2', '2.0.0', '3.0.0']);
114141
mockedCdn.putFileContent = sinon.stub();
115142
mockedCdn.putFileContent.resolves('ok');
116143
initialise();
117144
componentsCache.load().finally(done);
118145
});
119146

120147
it('should fetch the components.json', () => {
121-
expect(mockedCdn.getJson.calledOnce).to.be.true;
122148
expect(mockedCdn.getJson.args[0][0]).to.be.equal(
123149
'component/components.json'
124150
);
125151
});
126152

153+
it('should verify new versions', () => {
154+
expect(mockedCdn.getJson.calledThrice).to.be.true;
155+
expect(mockedCdn.getJson.args[1][0]).to.be.equal(
156+
'component/hello-world/2.0.0/package.json'
157+
);
158+
expect(mockedCdn.getJson.args[2][0]).to.be.equal(
159+
'component/hello-world/3.0.0/package.json'
160+
);
161+
});
162+
163+
it('should ignore corrupted versions and generate an error event', () => {
164+
expect(eventsHandlerStub.fire.called).to.be.true;
165+
expect(eventsHandlerStub.fire.args[0][0]).to.equal('error');
166+
expect(eventsHandlerStub.fire.args[0][1].code).to.equal(
167+
'corrupted_version'
168+
);
169+
expect(eventsHandlerStub.fire.args[0][1].message).to.contain(
170+
'hello-world'
171+
);
172+
expect(eventsHandlerStub.fire.args[0][1].message).to.contain('3.0.0');
173+
});
174+
127175
it('should scan for directories to fetch components and versions', () => {
128176
expect(mockedCdn.listSubDirectories.calledTwice).to.be.true;
129177
});

‎tsconfig.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
// "disableReferencedProjectLoad": true, /* Reduce the number of projects loaded automatically by TypeScript. */
1414

1515
/* Language and Environment */
16-
"target": "ES2018" /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */,
16+
"target": "ES2020" /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */,
1717
// "lib": [], /* Specify a set of bundled library declaration files that describe the target runtime environment. */
1818
// "jsx": "preserve", /* Specify what JSX code is generated. */
1919
// "experimentalDecorators": true, /* Enable experimental support for TC39 stage 2 draft decorators. */

0 commit comments

Comments
 (0)