Skip to content

Commit 577c28e

Browse files
ondrajodasondrajodas
authored
Merge pull request #130 from keboola/revert-129-AJDA-1176-ondra
Revert "[AJDA-1176] Skip orphaned manifests"
2 parents 308435b + 2220918 commit 577c28e

15 files changed

Lines changed: 258 additions & 372 deletions

File tree

‎phpstan-baseline.neon‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -840,6 +840,11 @@ parameters:
840840
count: 1
841841
path: tests/Keboola/GoogleAnalyticsExtractor/ApplicationTest.php
842842

843+
-
844+
message: "#^Method Keboola\\\\GoogleAnalyticsExtractor\\\\ApplicationTest\\:\\:assertManifestContainsColumns\\(\\) has parameter \\$expected with no value type specified in iterable type array\\.$#"
845+
count: 1
846+
path: tests/Keboola/GoogleAnalyticsExtractor/ApplicationTest.php
847+
843848
-
844849
message: "#^Method Keboola\\\\GoogleAnalyticsExtractor\\\\ApplicationTest\\:\\:getConfig\\(\\) return type has no value type specified in iterable type array\\.$#"
845850
count: 1
@@ -850,6 +855,11 @@ parameters:
850855
count: 1
851856
path: tests/Keboola/GoogleAnalyticsExtractor/ApplicationTest.php
852857

858+
-
859+
message: "#^Parameter \\#1 \\$json of function json_decode expects string, string\\|false given\\.$#"
860+
count: 1
861+
path: tests/Keboola/GoogleAnalyticsExtractor/ApplicationTest.php
862+
853863
-
854864
message: "#^Parameter \\#2 \\$array of static method PHPUnit\\\\Framework\\\\Assert\\:\\:assertArrayHasKey\\(\\) expects array\\|ArrayAccess, mixed given\\.$#"
855865
count: 2

‎src/Extractor/Extractor.php‎

Lines changed: 4 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -60,10 +60,10 @@ public function runProfiles(array $query, array $profiles): array
6060

6161
if (isset($query['query'])) {
6262
$outputCsv = $this->output->createReport($query);
63+
$this->output->createManifest($outputCsv->getFilename(), $query, ['id'], true);
6364
$this->logger->info(sprintf("Running query '%s'", $query['outputTable']));
6465

6566
$downloadedProfiles = false;
66-
$manifestCreated = false;
6767
foreach ($profiles as $profile) {
6868
$this->logger->info(sprintf('Profile "%s" export started.', $profile['id']));
6969
$apiQuery = $query;
@@ -130,16 +130,7 @@ public function runProfiles(array $query, array $profiles): array
130130
}
131131
}
132132

133-
$rowCount = $paginator->paginate($apiQuery, $report, $outputCsv);
134-
if ($rowCount > 0 && !$manifestCreated) {
135-
$this->output->createManifest(
136-
$outputCsv->getFilename(),
137-
$query,
138-
['id'],
139-
true,
140-
);
141-
$manifestCreated = true;
142-
}
133+
$paginator->paginate($apiQuery, $report, $outputCsv);
143134

144135
$status[$query['outputTable']][$profile['id']] = 'ok';
145136
}
@@ -168,10 +159,10 @@ public function runProperties(array $query, array $properties): array
168159
$query['query']['endpoint'] = 'properties';
169160

170161
$outputCsv = $this->output->createReport($query);
162+
$this->output->createManifest($outputCsv->getFilename(), $query, ['id'], true, 'idProperty');
171163
$this->logger->info(sprintf("Running query '%s'", $query['outputTable']));
172164

173165
$downloadedProperties = false;
174-
$manifestCreated = false;
175166
foreach ($properties as $property) {
176167
$this->logger->info(sprintf('Property "%s" export started.', $property['propertyName']));
177168
if (!empty($query['query']['viewId'])
@@ -208,17 +199,7 @@ public function runProperties(array $query, array $properties): array
208199
continue;
209200
}
210201

211-
$rowCount = $paginator->paginate($apiQuery, $report, $outputCsv);
212-
if ($rowCount > 0 && !$manifestCreated) {
213-
$this->output->createManifest(
214-
$outputCsv->getFilename(),
215-
$query,
216-
['id'],
217-
true,
218-
'idProperty',
219-
);
220-
$manifestCreated = true;
221-
}
202+
$paginator->paginate($apiQuery, $report, $outputCsv);
222203

223204
$status[$query['outputTable']][$property['propertyKey']] = 'ok';
224205
}

‎src/Extractor/Paginator/IPaginator.php‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,5 +12,5 @@ interface IPaginator
1212
{
1313
public function getOutput(): Output;
1414
public function getClient(): Client;
15-
public function paginate(array $query, array $report, CsvFile $csvFile): int;
15+
public function paginate(array $query, array $report, CsvFile $csvFile): void;
1616
}

‎src/Extractor/Paginator/ProfilesPaginator.php‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ public function getClient(): Client
3434
return $this->client;
3535
}
3636

37-
public function paginate(array $query, array $report, CsvFile $csvFile): int
37+
public function paginate(array $query, array $report, CsvFile $csvFile): void
3838
{
3939
$counter = 0;
4040
do {
@@ -60,8 +60,6 @@ public function paginate(array $query, array $report, CsvFile $csvFile): int
6060
}
6161
$query = $nextQuery;
6262
} while ($query);
63-
64-
return $counter;
6563
}
6664

6765
private function getStartIndex(string $link): string

‎src/Extractor/Paginator/PropertiesPaginator.php‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ public function setProperty(array $property): self
4444
return $this;
4545
}
4646

47-
public function paginate(array $query, array $report, CsvFile $csvFile): int
47+
public function paginate(array $query, array $report, CsvFile $csvFile): void
4848
{
4949
$localCounter = 0;
5050
do {
@@ -66,7 +66,5 @@ public function paginate(array $query, array $report, CsvFile $csvFile): int
6666

6767
$query = $nextQuery;
6868
} while ($report['totals'] > $localCounter);
69-
70-
return $localCounter;
7169
}
7270
}

‎tests/Keboola/GoogleAnalyticsExtractor/ApplicationTest.php‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,72 @@ private function getConfig(string $suffix = ''): array
5050
return $config;
5151
}
5252

53+
public function testAppRunDailyWalk(): void
54+
{
55+
$this->config = $this->getConfig('_antisampling');
56+
$this->runProcess();
57+
58+
$dailyWalk = $this->getManifestFiles('dailyWalk');
59+
Assert::assertEquals(1, count($dailyWalk));
60+
61+
foreach ($dailyWalk as $file) {
62+
/** @var $file SplFileInfo */
63+
$this->assertManifestContainsColumns($file->getPathname(), [
64+
'id',
65+
'idProfile',
66+
'date',
67+
'sourceMedium',
68+
'landingPagePath',
69+
'pageviews',
70+
]);
71+
}
72+
}
73+
74+
public function testAppRunAdaptive(): void
75+
{
76+
$this->config = $this->getConfig('_antisampling_adaptive');
77+
$this->runProcess();
78+
79+
$adaptive = $this->getManifestFiles('adaptive');
80+
Assert::assertEquals(1, count($adaptive));
81+
82+
foreach ($adaptive as $file) {
83+
/** @var $file SplFileInfo */
84+
$this->assertManifestContainsColumns($file->getPathname(), [
85+
'id',
86+
'idProfile',
87+
'date',
88+
'sourceMedium',
89+
'landingPagePath',
90+
'pageviews',
91+
]);
92+
}
93+
}
94+
95+
public function testAppRunMCF(): void
96+
{
97+
$this->config = $this->getConfig('_mcf');
98+
$this->runProcess();
99+
100+
$funnelFiles = $this->getManifestFiles('funnel');
101+
Assert::assertEquals(1, count($funnelFiles));
102+
103+
foreach ($funnelFiles as $file) {
104+
/** @var $file SplFileInfo */
105+
$this->assertManifestContainsColumns($file->getPathname(), [
106+
'id',
107+
'idProfile',
108+
'mcf:conversionDate',
109+
'mcf:sourcePath',
110+
'mcf:mediumPath',
111+
'mcf:sourceMedium',
112+
'mcf:totalConversions',
113+
'mcf:totalConversionValue',
114+
'mcf:assistedConversions',
115+
]);
116+
}
117+
}
118+
53119
public function testAppProfilesProperties(): void
54120
{
55121
$this->config = $this->getConfig('_empty');
@@ -117,6 +183,13 @@ private function getManifestFiles(string $queryName): Finder
117183
;
118184
}
119185

186+
private function assertManifestContainsColumns(string $pathname, array $expected): void
187+
{
188+
$manifest = (array) json_decode(file_get_contents($pathname), true, 512, JSON_THROW_ON_ERROR);
189+
Assert::assertArrayHasKey('columns', $manifest);
190+
Assert::assertEquals($expected, $manifest['columns']);
191+
}
192+
120193
public function appRunDataProvider(): Generator
121194
{
122195
yield 'configRow' => [

0 commit comments

Comments
 (0)