Skip to content

Commit 0992ce0

Browse files
author
Andy C
committed
[test/aports refactor] Move SQL to separate files
1 parent 0f31a51 commit 0992ce0

File tree

4 files changed

+130
-131
lines changed

4 files changed

+130
-131
lines changed

‎test/aports-diff.sql‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
-- Attach the source databases
2+
ATTACH DATABASE 'baseline/tables.db' AS baseline;
3+
ATTACH DATABASE 'osh-as-sh/tables.db' AS osh_as_sh;
4+
5+
.mode columns
6+
-- select * from packages;
7+
8+
create table diff as
9+
select
10+
b.pkg,
11+
cast(b.status as integer) as status1,
12+
"baseline" as baseline,
13+
"baseline/" || b.pkg_HREF as baseline_HREF,
14+
o.status as status2,
15+
"osh-as-sh" as osh_as_sh,
16+
"osh-as-sh/" || o.pkg_HREF as osh_as_sh_HREF,
17+
"diff" as diff,
18+
printf("error/%s.txt", b.pkg) as diff_HREF,
19+
"error" as error_grep,
20+
printf("error/%s.txt", b.pkg) as error_grep_HREF
21+
from baseline.packages b
22+
join osh_as_sh.packages o on b.pkg = o.pkg
23+
where b.status != o.status
24+
order by b.pkg;
25+
26+
-- Copied from above
27+
CREATE TABLE diff_schema AS
28+
SELECT
29+
name AS column_name,
30+
CASE
31+
WHEN UPPER(type) = "INTEGER" THEN "integer"
32+
WHEN UPPER(type) = "REAL" THEN "float"
33+
WHEN UPPER(type) = "TEXT" THEN "string"
34+
ELSE LOWER(type)
35+
END AS type
36+
FROM PRAGMA_TABLE_INFO("diff");
37+
38+
-- Detach databases
39+
DETACH DATABASE baseline;
40+
DETACH DATABASE osh_as_sh;

‎test/aports-html.sh‎

Lines changed: 9 additions & 131 deletions
Original file line numberDiff line numberDiff line change
@@ -168,26 +168,6 @@ drop table temp_import;
168168
"
169169
}
170170

171-
diff-report() {
172-
local dir=$REPORT_DIR/$EPOCH
173-
174-
{ typed-tsv-to-sql $dir/baseline/tasks.tsv baseline
175-
typed-tsv-to-sql $dir/osh-as-sh/tasks.tsv osh_as_sh
176-
echo '
177-
.mode column
178-
select count(*) from baseline;
179-
select count(*) from osh_as_sh;
180-
select * from pragma_table_info("baseline");
181-
select * from pragma_table_info("osh_as_sh");
182-
183-
-- 22 hours, but there was a big pause in the middle
184-
select ( max(end_time)-min(start_time) ) / 60 / 60 from baseline;
185-
186-
SELECT status, pkg FROM baseline WHERE status != 0;
187-
'
188-
} | sqlite3 :memory:
189-
}
190-
191171
my-rsync() {
192172
#rsync --archive --verbose --dry-run "$@"
193173
rsync --archive --verbose "$@"
@@ -219,80 +199,17 @@ make-package-table() {
219199
local db=$base_dir/$config/tables.db
220200
rm -f $db
221201

222-
{ typed-tsv-to-sql $base_dir/$config/tasks.tsv
223-
echo '
224-
.mode columns
225-
226-
-- select * from tasks limit 5;
227-
228-
-- annoying: you have to cast(x as real) for pragma_info to have type info
229-
create table packages as
230-
select status,
231-
elapsed_secs,
232-
cast( user_secs / elapsed_secs as real) as user_elapsed_ratio,
233-
cast( user_secs / sys_secs as real) as user_sys_ratio,
234-
cast(max_rss_KiB * 1024 / 1e6 as real) as max_rss_MB,
235-
pkg,
236-
pkg_HREF
237-
from tasks;
238-
239-
-- sqlite table schema -> foo.schema.tsv
240-
CREATE TABLE packages_schema AS
241-
SELECT
242-
name AS column_name,
243-
CASE
244-
WHEN UPPER(type) = "INTEGER" THEN "integer"
245-
WHEN UPPER(type) = "REAL" THEN "float"
246-
WHEN UPPER(type) = "TEXT" THEN "string"
247-
ELSE LOWER(type)
248-
END AS type
249-
FROM PRAGMA_TABLE_INFO("packages");
250-
251-
-- select * from packages_schema;
252-
253-
alter table packages_schema add column precision;
254-
255-
update packages_schema SET precision = 1 where column_name = "elapsed_secs";
256-
update packages_schema SET precision = 1 where column_name = "user_elapsed_ratio";
257-
update packages_schema SET precision = 1 where column_name = "user_sys_ratio";
258-
update packages_schema SET precision = 1 where column_name = "max_rss_MB";
259-
260-
-- Compute stats
261-
262-
CREATE TABLE metrics (
263-
id integer primary key check (id = 1), -- ensure only one row
264-
elapsed_minutes REAL NOT NULL,
265-
num_failures integer NOT NULL,
266-
num_tasks integer NOT NULL
267-
);
268-
269-
# dummy row
270-
insert into metrics values (1, -1.0, -1, -1);
271-
272-
update metrics
273-
set elapsed_minutes =
274-
(select ( max(end_time)-min(start_time) ) / 60 from tasks)
275-
where id = 1;
276-
277-
update metrics
278-
set num_failures =
279-
(select count(*) from tasks where status != 0)
280-
where id = 1;
281-
282-
update metrics
283-
set num_tasks =
284-
(select count(*) from tasks)
285-
where id = 1;
286-
'
287-
} | sqlite3 $db
288-
289-
sqlite3 $db >$base_dir/$config/packages.tsv <<EOF
202+
typed-tsv-to-sql $base_dir/$config/tasks.tsv | sqlite3 $db
203+
204+
sqlite3 -cmd '.mode columns' $db < test/aports-tasks.sql
205+
206+
sqlite3 $db >$base_dir/$config/packages.tsv <<'EOF'
290207
.mode tabs
291208
.headers on
292209
select * from packages;
293210
EOF
294211

295-
sqlite3 $db >$base_dir/$config/packages.schema.tsv <<EOF
212+
sqlite3 $db >$base_dir/$config/packages.schema.tsv <<'EOF'
296213
.mode tabs
297214
.headers on
298215
select * from packages_schema;
@@ -350,50 +267,11 @@ make-diff-db() {
350267

351268
local db=$name.db
352269

270+
local sql=$PWD/test/aports-diff.sql
271+
353272
pushd $base_dir
354273
rm -f $db
355-
sqlite3 $db <<EOF
356-
-- Attach the source databases
357-
ATTACH DATABASE 'baseline/tables.db' AS baseline;
358-
ATTACH DATABASE 'osh-as-sh/tables.db' AS osh_as_sh;
359-
360-
.mode columns
361-
-- select * from packages;
362-
363-
create table diff as
364-
select
365-
b.pkg,
366-
cast(b.status as integer) as status1,
367-
"baseline" as baseline,
368-
"baseline/" || b.pkg_HREF as baseline_HREF,
369-
o.status as status2,
370-
"osh-as-sh" as osh_as_sh,
371-
"osh-as-sh/" || o.pkg_HREF as osh_as_sh_HREF,
372-
"diff" as diff,
373-
printf("error/%s.txt", b.pkg) as diff_HREF,
374-
"error" as error_grep,
375-
printf("error/%s.txt", b.pkg) as error_grep_HREF
376-
from baseline.packages b
377-
join osh_as_sh.packages o on b.pkg = o.pkg
378-
where b.status != o.status
379-
order by b.pkg;
380-
381-
-- Copied from above
382-
CREATE TABLE diff_schema AS
383-
SELECT
384-
name AS column_name,
385-
CASE
386-
WHEN UPPER(type) = "INTEGER" THEN "integer"
387-
WHEN UPPER(type) = "REAL" THEN "float"
388-
WHEN UPPER(type) = "TEXT" THEN "string"
389-
ELSE LOWER(type)
390-
END AS type
391-
FROM PRAGMA_TABLE_INFO("diff");
392-
393-
-- Detach databases
394-
DETACH DATABASE baseline;
395-
DETACH DATABASE osh_as_sh;
396-
EOF
274+
sqlite3 $db < $sql
397275

398276
sqlite3 $db >$name.tsv <<EOF
399277
.mode tabs

‎test/aports-old.sh‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,4 +121,25 @@ ATTACH DATABASE 'osh-as-sh/packages.db' AS osh_as_sh;
121121
EOF
122122
}
123123

124+
diff-report() {
125+
local dir=$REPORT_DIR/$EPOCH
126+
127+
{ typed-tsv-to-sql $dir/baseline/tasks.tsv baseline
128+
typed-tsv-to-sql $dir/osh-as-sh/tasks.tsv osh_as_sh
129+
echo '
130+
.mode column
131+
select count(*) from baseline;
132+
select count(*) from osh_as_sh;
133+
select * from pragma_table_info("baseline");
134+
select * from pragma_table_info("osh_as_sh");
135+
136+
-- 22 hours, but there was a big pause in the middle
137+
select ( max(end_time)-min(start_time) ) / 60 / 60 from baseline;
138+
139+
SELECT status, pkg FROM baseline WHERE status != 0;
140+
'
141+
} | sqlite3 :memory:
142+
}
143+
144+
124145
task-five "$@"

‎test/aports-tasks.sql‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
-- select * from tasks limit 5;
2+
3+
-- annoying: you have to cast(x as real) for pragma_info to have type info
4+
create table packages as
5+
select status,
6+
elapsed_secs,
7+
cast( user_secs / elapsed_secs as real) as user_elapsed_ratio,
8+
cast( user_secs / sys_secs as real) as user_sys_ratio,
9+
cast(max_rss_KiB * 1024 / 1e6 as real) as max_rss_MB,
10+
pkg,
11+
pkg_HREF
12+
from tasks;
13+
14+
-- sqlite table schema -> foo.schema.tsv
15+
CREATE TABLE packages_schema AS
16+
SELECT
17+
name AS column_name,
18+
CASE
19+
WHEN UPPER(type) = "INTEGER" THEN "integer"
20+
WHEN UPPER(type) = "REAL" THEN "float"
21+
WHEN UPPER(type) = "TEXT" THEN "string"
22+
ELSE LOWER(type)
23+
END AS type
24+
FROM PRAGMA_TABLE_INFO("packages");
25+
26+
-- select * from packages_schema;
27+
28+
alter table packages_schema add column precision;
29+
30+
update packages_schema SET precision = 1 where column_name = "elapsed_secs";
31+
update packages_schema SET precision = 1 where column_name = "user_elapsed_ratio";
32+
update packages_schema SET precision = 1 where column_name = "user_sys_ratio";
33+
update packages_schema SET precision = 1 where column_name = "max_rss_MB";
34+
35+
-- Compute stats
36+
37+
CREATE TABLE metrics (
38+
id integer primary key check (id = 1), -- ensure only one row
39+
elapsed_minutes REAL NOT NULL,
40+
num_failures integer NOT NULL,
41+
num_tasks integer NOT NULL
42+
);
43+
44+
# dummy row
45+
insert into metrics values (1, -1.0, -1, -1);
46+
47+
update metrics
48+
set elapsed_minutes =
49+
(select ( max(end_time)-min(start_time) ) / 60 from tasks)
50+
where id = 1;
51+
52+
update metrics
53+
set num_failures =
54+
(select count(*) from tasks where status != 0)
55+
where id = 1;
56+
57+
update metrics
58+
set num_tasks =
59+
(select count(*) from tasks)
60+
where id = 1;

0 commit comments

Comments
 (0)