fix(src/queries): update queries (#843)

queries can be updated to just return table; #844
adobe · Mar 30, 2023 · 355648f · 355648f
1 parent 718ab20
commit 355648f
Show file tree

Hide file tree

Showing 20 changed files with 1,295 additions and 1,140 deletions.
diff --git a/package.json b/package.json
@@ -105,7 +105,8 @@
       "src/queries/rum-sources-targets.sql",
       "src/queries/rum-content-requests.sql",
       "src/queries/daily-rum.sql",
-      "src/queries/live-sites.sql",
+      "src/queries/site4s.sql",
+      "src/queries/github-prs.sql",
       "src/queries/rum-checkpoint-cwv-correlation.sql"
     ]
   }

diff --git a/src/queries/blobs-served-hlx3.sql b/src/queries/blobs-served-hlx3.sql
@@ -17,30 +17,70 @@
 --- limit: 100
 --- offset: 0
 SELECT * FROM (
-SELECT
-    IF(req_http_X_Owner != "", req_http_X_Owner, REGEXP_EXTRACT(req_http_host, "[^\\-\\.]+--([^\\-\\.]+)\\.", 1))  AS owner,
-    IF(req_http_X_Repo != "", req_http_X_Repo,
+    SELECT
+        REQ_HTTP_X_URL AS URL,
         IF(
-            ARRAY_LENGTH(SPLIT(SPLIT(req_http_host, '.')[OFFSET(0)], '--')) = 3,
-            SPLIT(SPLIT(req_http_host, '.')[OFFSET(0)], '--')[OFFSET(1)],
-            SPLIT(SPLIT(req_http_host, '.')[OFFSET(0)], '--')[OFFSET(0)]
-                ))AS repo,
-    REGEXP_EXTRACT(req_http_X_URL, "_([0-9a-f]+)\\.[a-z]+\\??", 1) AS id,
-    req_http_X_URL AS url,
-    REGEXP_EXTRACT(req_http_X_URL, "_[0-9a-f]+\\.([a-z]+)\\??", 1) AS format,
-    TIMESTAMP_MICROS(CAST(MAX(time_start_usec) AS INT64)) AS latestreq,
-    TIMESTAMP_MICROS(CAST(MIN(time_start_usec) AS INT64)) AS earliestreq,
-    COUNT(time_start_usec) AS requests,
-    MAX(req_http_host) AS host,
-FROM `helix_logging_7TvULgs0Xnls4q3R8tawdg.requests*`
-WHERE req_http_X_URL LIKE "%_media%"
-    AND status_code = CAST(@status AS STRING) AND
-    _TABLE_SUFFIX <= CONCAT(CAST(EXTRACT(YEAR FROM CURRENT_TIMESTAMP()) AS String), LPAD(CAST(EXTRACT(MONTH FROM CURRENT_TIMESTAMP()) AS String), 2, "0")) AND
-    _TABLE_SUFFIX >= CONCAT(CAST(EXTRACT(YEAR FROM TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL @fromHours HOUR)) AS String), LPAD(CAST(EXTRACT(MONTH FROM TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL @fromHours HOUR)) AS String), 2, "0")) AND
-    time_start_usec > CAST(UNIX_MICROS(TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL @fromHours HOUR)) AS STRING) AND
-    time_start_usec < CAST(UNIX_MICROS(TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 0 DAY)) AS STRING)
-GROUP BY owner, repo, id, format, url
-ORDER BY requests DESC
+            REQ_HTTP_X_OWNER != "",
+            REQ_HTTP_X_OWNER,
+            REGEXP_EXTRACT(REQ_HTTP_HOST, "[^\\-\\.]+--([^\\-\\.]+)\\.", 1)
+        ) AS OWNER,
+        IF(REQ_HTTP_X_REPO != "", REQ_HTTP_X_REPO,
+            IF(
+                ARRAY_LENGTH(
+                    SPLIT(SPLIT(REQ_HTTP_HOST, ".") [OFFSET(0)], "--")
+                ) = 3,
+                SPLIT(SPLIT(REQ_HTTP_HOST, ".") [OFFSET(0)], "--") [OFFSET(1)],
+                SPLIT(SPLIT(REQ_HTTP_HOST, ".") [OFFSET(0)], "--") [OFFSET(0)]
+            )) AS REPO,
+        REGEXP_EXTRACT(REQ_HTTP_X_URL, "_([0-9a-f]+)\\.[a-z]+\\??", 1) AS ID,
+        REGEXP_EXTRACT(
+            REQ_HTTP_X_URL, "_[0-9a-f]+\\.([a-z]+)\\??", 1
+        ) AS FORMAT,
+        TIMESTAMP_MICROS(CAST(MAX(TIME_START_USEC) AS INT64)) AS LATESTREQ,
+        TIMESTAMP_MICROS(CAST(MIN(TIME_START_USEC) AS INT64)) AS EARLIESTREQ,
+        COUNT(TIME_START_USEC) AS REQUESTS,
+        MAX(REQ_HTTP_HOST) AS HOST
+    FROM `helix_logging_7TvULgs0Xnls4q3R8tawdg.requests*`
+    WHERE REQ_HTTP_X_URL LIKE "%_media%"
+        AND STATUS_CODE = CAST(@status AS STRING)
+        AND _TABLE_SUFFIX <= CONCAT(
+            CAST(EXTRACT(YEAR FROM CURRENT_TIMESTAMP()) AS String),
+            LPAD(
+                CAST(EXTRACT(MONTH FROM CURRENT_TIMESTAMP()) AS String), 2, "0"
+            )
+        )
+        AND _TABLE_SUFFIX >= CONCAT(
+            CAST(
+                EXTRACT(
+                    YEAR FROM TIMESTAMP_SUB(
+                        CURRENT_TIMESTAMP(), INTERVAL @fromHours HOUR
+                    )
+                ) AS String
+            ),
+            LPAD(
+                CAST(
+                    EXTRACT(
+                        MONTH FROM TIMESTAMP_SUB(
+                            CURRENT_TIMESTAMP(), INTERVAL @fromHours HOUR
+                        )
+                    ) AS String
+                ),
+                2,
+                "0"
+            )
+        )
+        AND TIME_START_USEC > CAST(
+            UNIX_MICROS(
+                TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL @fromHours HOUR)
+            ) AS STRING
+        )
+        AND TIME_START_USEC < CAST(
+            UNIX_MICROS(
+                TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 0 DAY)
+            ) AS STRING
+        )
+    GROUP BY OWNER, REPO, ID, FORMAT, URL
+    ORDER BY REQUESTS DESC
 )
-WHERE id IS NOT NULL AND format IS NOT NULL
+WHERE ID IS NOT NULL AND FORMAT IS NOT NULL
 LIMIT @limit OFFSET @offset
diff --git a/src/queries/content-power-curve.sql b/src/queries/content-power-curve.sql
@@ -6,47 +6,76 @@
 --- offset: 0
 --- domain: -
 WITH visits AS (
-  SELECT 
-    url, 
-    TIMESTAMP_TRUNC(time, DAY) AS time, 
-    id, 
-    pageviews AS weight
-  FROM helix_rum.CLUSTER_PAGEVIEWS(
-    @domain,
-    CAST(@offset AS INT64),
-    CAST(@interval AS INT64) + 1,
-    '2022-02-01', # not used, start date
-    '2022-05-28', # not used, end date
-    'GMT', # timezone
-    'all', # device class
-    '-' # not used, generation
-  )
+    SELECT
+        url,
+        id,
+        pageviews AS weight,
+        TIMESTAMP_TRUNC(time, DAY) AS time
+    FROM helix_rum.CLUSTER_PAGEVIEWS(
+        @domain,
+        CAST(@offset AS INT64),
+        CAST(@interval AS INT64) + 1,
+        '2022-02-01', # not used, start date
+        '2022-05-28', # not used, end date
+        'GMT', # timezone
+        'all', # device class
+        '-' # not used, generation
+    )
 ),
+
 urldays AS (
-  SELECT time, url, COUNT(id) AS events, SUM(weight) AS visits FROM visits # FULL JOIN days ON (days.time = visits.time)
-  GROUP BY time, url
+    # FULL JOIN days ON (days.time = visits.time)
+    SELECT
+        time,
+        url,
+        COUNT(id) AS events,
+        SUM(weight) AS visits
+    FROM visits
+    GROUP BY time, url
 ),
+
 steps AS (
-  SELECT time, url, events, visits, 
-  TIMESTAMP_DIFF(time, LAG(time) OVER(PARTITION BY url ORDER BY time), DAY) AS step 
-  FROM urldays
+    SELECT
+        time,
+        url,
+        events,
+        visits,
+        TIMESTAMP_DIFF(
+            time, LAG(time) OVER(PARTITION BY url ORDER BY time), DAY
+        ) AS step
+    FROM urldays
 ),
+
 chains AS (
-  SELECT time, url, events, visits, steps, 
-      COUNTIF(step = 1) OVER(PARTITION BY url ORDER BY time) AS chain
-      FROM steps
+    SELECT
+        time,
+        url,
+        events,
+        visits,
+        steps,
+        COUNTIF(step = 1) OVER(PARTITION BY url ORDER BY time) AS chain
+    FROM steps
 ),
+
 urlchains AS (
-  SELECT url, time, chain, events, visits FROM chains
-  ORDER BY chain DESC
+    SELECT
+        url,
+        time,
+        chain,
+        events,
+        visits
+    FROM chains
+    ORDER BY chain DESC
 ),
+
 powercurve AS (
-  SELECT 
-    MAX(chain) AS persistence, 
-    count(url) AS reach
-  FROM urlchains
-  GROUP BY chain
-  ORDER BY MAX(chain) ASC
-  LIMIT 31 OFFSET 1
+    SELECT
+        MAX(chain) AS persistence,
+        COUNT(url) AS reach
+    FROM urlchains
+    GROUP BY chain
+    ORDER BY MAX(chain) ASC
+    LIMIT 31 OFFSET 1
 )
-SELECT * FROM powercurve
+
+SELECT * FROM powercurve
diff --git a/src/queries/crux-trends.sql b/src/queries/crux-trends.sql
@@ -5,96 +5,102 @@
 --- then: 2201
 --- now: 2205
 WITH lcp_now AS (
-  SELECT SUM(lcp.density) AS good_lcp_now
-  FROM
-    `chrome-ux-report.all.20*`,
-    UNNEST(largest_contentful_paint.histogram.bin) AS lcp
-  WHERE
-    origin = CONCAT("https://", @domain)
-    AND lcp.start < 2500
-    AND _table_suffix = @now
+    SELECT SUM(lcp.density) AS good_lcp_now
+    FROM
+        `chrome-ux-report.all.20*`,
+        UNNEST(largest_contentful_paint.histogram.bin) AS lcp
+    WHERE
+        origin = CONCAT("https://", @domain)
+        AND lcp.start < 2500
+        AND _table_suffix = @now
 ),
 
 lcp_then AS (
-  SELECT SUM(lcp.density) AS good_lcp_then
-  FROM
-    `chrome-ux-report.all.20*`,
-    UNNEST(largest_contentful_paint.histogram.bin) AS lcp
-  WHERE
-    origin = CONCAT("https://", @domain)
-    AND lcp.start < 2500
-    AND _table_suffix = @then
+    SELECT SUM(lcp.density) AS good_lcp_then
+    FROM
+        `chrome-ux-report.all.20*`,
+        UNNEST(largest_contentful_paint.histogram.bin) AS lcp
+    WHERE
+        origin = CONCAT("https://", @domain)
+        AND lcp.start < 2500
+        AND _table_suffix = @then
 ),
 
 cls_now AS (
-  SELECT SUM(cls.density) AS good_cls_now
-  FROM
-    `chrome-ux-report.all.20*`,
-    UNNEST(layout_instability.cumulative_layout_shift.histogram.bin) AS cls
-  WHERE
-    origin = CONCAT("https://", @domain)
-    AND cls.start < 0.1
-    AND _table_suffix = @now
+    SELECT SUM(cls.density) AS good_cls_now
+    FROM
+        `chrome-ux-report.all.20*`,
+        UNNEST(layout_instability.cumulative_layout_shift.histogram.bin) AS cls
+    WHERE
+        origin = CONCAT("https://", @domain)
+        AND cls.start < 0.1
+        AND _table_suffix = @now
 ),
 
 cls_then AS (
-  SELECT SUM(cls.density) AS good_cls_then
-  FROM
-    `chrome-ux-report.all.20*`,
-    UNNEST(layout_instability.cumulative_layout_shift.histogram.bin) AS cls
-  WHERE
-    origin = CONCAT("https://", @domain)
-    AND cls.start < 0.1
-    AND _table_suffix = @then
+    SELECT SUM(cls.density) AS good_cls_then
+    FROM
+        `chrome-ux-report.all.20*`,
+        UNNEST(layout_instability.cumulative_layout_shift.histogram.bin) AS cls
+    WHERE
+        origin = CONCAT("https://", @domain)
+        AND cls.start < 0.1
+        AND _table_suffix = @then
 ),
 
 fid_now AS (
-  SELECT SUM(fid.density) AS good_fid_now
-  FROM
-    `chrome-ux-report.all.20*`,
-    UNNEST(first_input.delay.histogram.bin) AS fid
-  WHERE
-    origin = CONCAT("https://", @domain)
-    AND fid.start < 100
-    AND _table_suffix = @now
+    SELECT SUM(fid.density) AS good_fid_now
+    FROM
+        `chrome-ux-report.all.20*`,
+        UNNEST(first_input.delay.histogram.bin) AS fid
+    WHERE
+        origin = CONCAT("https://", @domain)
+        AND fid.start < 100
+        AND _table_suffix = @now
 ),
 
 fid_then AS (
-  SELECT SUM(fid.density) AS good_fid_then
-  FROM
-    `chrome-ux-report.all.20*`,
-    UNNEST(first_input.delay.histogram.bin) AS fid
-  WHERE
-    origin = CONCAT("https://", @domain)
-    AND fid.start < 100
-    AND _table_suffix = @then
+    SELECT SUM(fid.density) AS good_fid_then
+    FROM
+        `chrome-ux-report.all.20*`,
+        UNNEST(first_input.delay.histogram.bin) AS fid
+    WHERE
+        origin = CONCAT("https://", @domain)
+        AND fid.start < 100
+        AND _table_suffix = @then
 ),
 
 comparison AS (
-  SELECT
-    cls_now.good_cls_now,
-    cls_then.good_cls_then,
-    fid_now.good_fid_now,
-    fid_then.good_fid_then,
-    lcp_now.good_lcp_now,
-    lcp_then.good_lcp_then
-  FROM lcp_now, cls_now, fid_now, lcp_then, cls_then, fid_then
+    SELECT
+        cls_now.good_cls_now,
+        cls_then.good_cls_then,
+        fid_now.good_fid_now,
+        fid_then.good_fid_then,
+        lcp_now.good_lcp_now,
+        lcp_then.good_lcp_then
+    FROM lcp_now, cls_now, fid_now, lcp_then, cls_then, fid_then
 )
 
 SELECT
-  STRUCT(
-    ROUND(good_lcp_now * 100) AS now,
-    ROUND(good_lcp_then * 100) AS before,
-    ROUND(100 * (good_lcp_now - good_lcp_then) / good_lcp_then) AS improvement
-  ) AS lcp,
-  STRUCT(
-    ROUND(good_fid_now * 100) AS now,
-    ROUND(good_fid_then * 100) AS before,
-    ROUND(100 * (good_fid_now - good_fid_then) / good_fid_then) AS improvement
-  ) AS fid,
-  STRUCT(
-    ROUND(good_cls_now * 100) AS now,
-    ROUND(good_cls_then * 100) AS before,
-    ROUND(100 * (good_cls_now - good_cls_then) / good_cls_then) AS improvement
-  ) AS cls
+    STRUCT(
+        ROUND(good_lcp_now * 100) AS now,
+        ROUND(good_lcp_then * 100) AS before,
+        ROUND(
+            100 * (good_lcp_now - good_lcp_then) / good_lcp_then
+        ) AS improvement
+    ) AS lcp,
+    STRUCT(
+        ROUND(good_fid_now * 100) AS now,
+        ROUND(good_fid_then * 100) AS before,
+        ROUND(
+            100 * (good_fid_now - good_fid_then) / good_fid_then
+        ) AS improvement
+    ) AS fid,
+    STRUCT(
+        ROUND(good_cls_now * 100) AS now,
+        ROUND(good_cls_then * 100) AS before,
+        ROUND(
+            100 * (good_cls_now - good_cls_then) / good_cls_then
+        ) AS improvement
+    ) AS cls
 FROM comparison