Skip to content

Commit 308435b

Browse files
ondrajodasondrajodas
authored
Merge pull request #129 from keboola/AJDA-1176-ondra
[AJDA-1176] Skip orphaned manifests
2 parents f06df37 + 11daa41 commit 308435b

15 files changed

Lines changed: 372 additions & 258 deletions

File tree

‎phpstan-baseline.neon‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -840,11 +840,6 @@ parameters:
840840
count: 1
841841
path: tests/Keboola/GoogleAnalyticsExtractor/ApplicationTest.php
842842

843-
-
844-
message: "#^Method Keboola\\\\GoogleAnalyticsExtractor\\\\ApplicationTest\\:\\:assertManifestContainsColumns\\(\\) has parameter \\$expected with no value type specified in iterable type array\\.$#"
845-
count: 1
846-
path: tests/Keboola/GoogleAnalyticsExtractor/ApplicationTest.php
847-
848843
-
849844
message: "#^Method Keboola\\\\GoogleAnalyticsExtractor\\\\ApplicationTest\\:\\:getConfig\\(\\) return type has no value type specified in iterable type array\\.$#"
850845
count: 1
@@ -855,11 +850,6 @@ parameters:
855850
count: 1
856851
path: tests/Keboola/GoogleAnalyticsExtractor/ApplicationTest.php
857852

858-
-
859-
message: "#^Parameter \\#1 \\$json of function json_decode expects string, string\\|false given\\.$#"
860-
count: 1
861-
path: tests/Keboola/GoogleAnalyticsExtractor/ApplicationTest.php
862-
863853
-
864854
message: "#^Parameter \\#2 \\$array of static method PHPUnit\\\\Framework\\\\Assert\\:\\:assertArrayHasKey\\(\\) expects array\\|ArrayAccess, mixed given\\.$#"
865855
count: 2

‎src/Extractor/Extractor.php‎

Lines changed: 23 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -60,10 +60,10 @@ public function runProfiles(array $query, array $profiles): array
6060

6161
if (isset($query['query'])) {
6262
$outputCsv = $this->output->createReport($query);
63-
$this->output->createManifest($outputCsv->getFilename(), $query, ['id'], true);
6463
$this->logger->info(sprintf("Running query '%s'", $query['outputTable']));
6564

6665
$downloadedProfiles = false;
66+
$manifestCreated = false;
6767
foreach ($profiles as $profile) {
6868
$this->logger->info(sprintf('Profile "%s" export started.', $profile['id']));
6969
$apiQuery = $query;
@@ -130,7 +130,16 @@ public function runProfiles(array $query, array $profiles): array
130130
}
131131
}
132132

133-
$paginator->paginate($apiQuery, $report, $outputCsv);
133+
$rowCount = $paginator->paginate($apiQuery, $report, $outputCsv);
134+
if ($rowCount > 0 && !$manifestCreated) {
135+
$this->output->createManifest(
136+
$outputCsv->getFilename(),
137+
$query,
138+
['id'],
139+
true,
140+
);
141+
$manifestCreated = true;
142+
}
134143

135144
$status[$query['outputTable']][$profile['id']] = 'ok';
136145
}
@@ -159,10 +168,10 @@ public function runProperties(array $query, array $properties): array
159168
$query['query']['endpoint'] = 'properties';
160169

161170
$outputCsv = $this->output->createReport($query);
162-
$this->output->createManifest($outputCsv->getFilename(), $query, ['id'], true, 'idProperty');
163171
$this->logger->info(sprintf("Running query '%s'", $query['outputTable']));
164172

165173
$downloadedProperties = false;
174+
$manifestCreated = false;
166175
foreach ($properties as $property) {
167176
$this->logger->info(sprintf('Property "%s" export started.', $property['propertyName']));
168177
if (!empty($query['query']['viewId'])
@@ -199,7 +208,17 @@ public function runProperties(array $query, array $properties): array
199208
continue;
200209
}
201210

202-
$paginator->paginate($apiQuery, $report, $outputCsv);
211+
$rowCount = $paginator->paginate($apiQuery, $report, $outputCsv);
212+
if ($rowCount > 0 && !$manifestCreated) {
213+
$this->output->createManifest(
214+
$outputCsv->getFilename(),
215+
$query,
216+
['id'],
217+
true,
218+
'idProperty',
219+
);
220+
$manifestCreated = true;
221+
}
203222

204223
$status[$query['outputTable']][$property['propertyKey']] = 'ok';
205224
}

‎src/Extractor/Paginator/IPaginator.php‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,5 +12,5 @@ interface IPaginator
1212
{
1313
public function getOutput(): Output;
1414
public function getClient(): Client;
15-
public function paginate(array $query, array $report, CsvFile $csvFile): void;
15+
public function paginate(array $query, array $report, CsvFile $csvFile): int;
1616
}

‎src/Extractor/Paginator/ProfilesPaginator.php‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ public function getClient(): Client
3434
return $this->client;
3535
}
3636

37-
public function paginate(array $query, array $report, CsvFile $csvFile): void
37+
public function paginate(array $query, array $report, CsvFile $csvFile): int
3838
{
3939
$counter = 0;
4040
do {
@@ -60,6 +60,8 @@ public function paginate(array $query, array $report, CsvFile $csvFile): void
6060
}
6161
$query = $nextQuery;
6262
} while ($query);
63+
64+
return $counter;
6365
}
6466

6567
private function getStartIndex(string $link): string

‎src/Extractor/Paginator/PropertiesPaginator.php‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ public function setProperty(array $property): self
4444
return $this;
4545
}
4646

47-
public function paginate(array $query, array $report, CsvFile $csvFile): void
47+
public function paginate(array $query, array $report, CsvFile $csvFile): int
4848
{
4949
$localCounter = 0;
5050
do {
@@ -66,5 +66,7 @@ public function paginate(array $query, array $report, CsvFile $csvFile): void
6666

6767
$query = $nextQuery;
6868
} while ($report['totals'] > $localCounter);
69+
70+
return $localCounter;
6971
}
7072
}

‎tests/Keboola/GoogleAnalyticsExtractor/ApplicationTest.php‎

Lines changed: 0 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -50,72 +50,6 @@ private function getConfig(string $suffix = ''): array
5050
return $config;
5151
}
5252

53-
public function testAppRunDailyWalk(): void
54-
{
55-
$this->config = $this->getConfig('_antisampling');
56-
$this->runProcess();
57-
58-
$dailyWalk = $this->getManifestFiles('dailyWalk');
59-
Assert::assertEquals(1, count($dailyWalk));
60-
61-
foreach ($dailyWalk as $file) {
62-
/** @var $file SplFileInfo */
63-
$this->assertManifestContainsColumns($file->getPathname(), [
64-
'id',
65-
'idProfile',
66-
'date',
67-
'sourceMedium',
68-
'landingPagePath',
69-
'pageviews',
70-
]);
71-
}
72-
}
73-
74-
public function testAppRunAdaptive(): void
75-
{
76-
$this->config = $this->getConfig('_antisampling_adaptive');
77-
$this->runProcess();
78-
79-
$adaptive = $this->getManifestFiles('adaptive');
80-
Assert::assertEquals(1, count($adaptive));
81-
82-
foreach ($adaptive as $file) {
83-
/** @var $file SplFileInfo */
84-
$this->assertManifestContainsColumns($file->getPathname(), [
85-
'id',
86-
'idProfile',
87-
'date',
88-
'sourceMedium',
89-
'landingPagePath',
90-
'pageviews',
91-
]);
92-
}
93-
}
94-
95-
public function testAppRunMCF(): void
96-
{
97-
$this->config = $this->getConfig('_mcf');
98-
$this->runProcess();
99-
100-
$funnelFiles = $this->getManifestFiles('funnel');
101-
Assert::assertEquals(1, count($funnelFiles));
102-
103-
foreach ($funnelFiles as $file) {
104-
/** @var $file SplFileInfo */
105-
$this->assertManifestContainsColumns($file->getPathname(), [
106-
'id',
107-
'idProfile',
108-
'mcf:conversionDate',
109-
'mcf:sourcePath',
110-
'mcf:mediumPath',
111-
'mcf:sourceMedium',
112-
'mcf:totalConversions',
113-
'mcf:totalConversionValue',
114-
'mcf:assistedConversions',
115-
]);
116-
}
117-
}
118-
11953
public function testAppProfilesProperties(): void
12054
{
12155
$this->config = $this->getConfig('_empty');
@@ -183,13 +117,6 @@ private function getManifestFiles(string $queryName): Finder
183117
;
184118
}
185119

186-
private function assertManifestContainsColumns(string $pathname, array $expected): void
187-
{
188-
$manifest = (array) json_decode(file_get_contents($pathname), true, 512, JSON_THROW_ON_ERROR);
189-
Assert::assertArrayHasKey('columns', $manifest);
190-
Assert::assertEquals($expected, $manifest['columns']);
191-
}
192-
193120
public function appRunDataProvider(): Generator
194121
{
195122
yield 'configRow' => [

0 commit comments

Comments
 (0)