feat: Task 8, duck DB with local file

Author: lfoppiano

CC-MAIN-2024-22.warc.paths.gz

@@ -44,14 +44,15 @@ CC-MAIN-2024-22.warc.paths.gz:
 # 	@echo "warning! 300 gigabyte download"
 # 	python duck.py local_files
 #
-# duck_ccf_local_files:
-# 	@echo "warning! only works on Common Crawl Foundadtion's development machine"
-# 	python duck.py ccf_local_files
-#
-# duck_cloudfront:
-# 	@echo "warning! this might take 1-10 minutes"
-# 	python duck.py cloudfront
-#
+duck_ccf_local_files: build
+	@echo "warning! only works on Common Crawl Foundadtion's development machine"
+	mvn -q exec:java -Dexec.mainClass=org.commoncrawl.whirlwind.Duck -Dexec.args"ccf_local_files"
+
+duck_cloudfront: build
+	@echo "warning! this might take 1-10 minutes"
+	mvn -q exec:java -Dexec.mainClass=org.commoncrawl.whirlwind.Duck -Dexec.args"cloudfront"
+
+
 ensure_jwarc:
 	@echo "Ensuring JWarc JAR is present"
 	@if [ ! -f jwarc.jar ] ; then \

Makefile

@@ -453,82 +453,63 @@ On a machine with a 1 gigabit network connection and many cores, this should tak
   <summary>Click to view output</summary>
 
 ```
-warning! this might take 1-10 minutes
-python duck.py cloudfront
-total records for crawl: CC-MAIN-2024-22
-┌──────────────┐
-│ count_star() │
-│    int64     │
-├──────────────┤
-│   2709877975 │
-└──────────────┘
-
-our one row
-┌──────────────────────┬──────────────────────┬──────────────────┬───┬──────────────────┬─────────────────┬─────────┐
-│     url_surtkey      │         url          │  url_host_name   │ … │   warc_segment   │      crawl      │ subset  │
-│       varchar        │       varchar        │     varchar      │   │     varchar      │     varchar     │ varchar │
-├──────────────────────┼──────────────────────┼──────────────────┼───┼──────────────────┼─────────────────┼─────────┤
-│ org,wikipedia,an)/…  │ https://an.wikiped…  │ an.wikipedia.org │ … │ 1715971057216.39 │ CC-MAIN-2024-22 │ warc    │
-├──────────────────────┴──────────────────────┴──────────────────┴───┴──────────────────┴─────────────────┴─────────┤
-│ 1 rows                                                                                       32 columns (6 shown) │
-└───────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘
-
-writing our one row to a local parquet file, whirlwind.parquet
-total records for local whirlwind.parquet should be 1
-┌──────────────┐
-│ count_star() │
-│    int64     │
-├──────────────┤
-│            1 │
-└──────────────┘
-
-our one row, locally
-┌──────────────────────┬──────────────────────┬──────────────────┬───┬──────────────────┬─────────────────┬─────────┐
-│     url_surtkey      │         url          │  url_host_name   │ … │   warc_segment   │      crawl      │ subset  │
-│       varchar        │       varchar        │     varchar      │   │     varchar      │     varchar     │ varchar │
-├──────────────────────┼──────────────────────┼──────────────────┼───┼──────────────────┼─────────────────┼─────────┤
-│ org,wikipedia,an)/…  │ https://an.wikiped…  │ an.wikipedia.org │ … │ 1715971057216.39 │ CC-MAIN-2024-22 │ warc    │
-├──────────────────────┴──────────────────────┴──────────────────┴───┴──────────────────┴─────────────────┴─────────┤
-│ 1 rows                                                                                       32 columns (6 shown) │
-└───────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘
-
-complete row:
+Using algorithm: cloudfront
+Total records for crawl: CC-MAIN-2024-22
+100% ▕████████████████████████████████████████████████████████████▏ 
+2709877975
+
+Our one row:
+100% ▕████████████████████████████████████████████████████████████▏ 
+url_surtkey | url | url_host_name | url_host_tld | url_host_2nd_last_part | url_host_3rd_last_part | url_host_4th_last_part | url_host_5th_last_part | url_host_registry_suffix | url_host_registered_domain | url_host_private_suffix | url_host_private_domain | url_host_name_reversed | url_protocol | url_port | url_path | url_query | fetch_time | fetch_status | fetch_redirect | content_digest | content_mime_type | content_mime_detected | content_charset | content_languages | content_truncated | warc_filename | warc_record_offset | warc_record_length | warc_segment | crawl | subset
+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+org,wikipedia,an)/wiki/escopete | https://an.wikipedia.org/wiki/Escopete | an.wikipedia.org | org | wikipedia | an | NULL | NULL | org | wikipedia.org | org | wikipedia.org | org.wikipedia.an | https | NULL | /wiki/Escopete | NULL | 2024-05-18T01:58:10Z | 200 | NULL | RY7PLBUFQNI2FFV5FTUQK72W6SNPXLQU | text/html | text/html | UTF-8 | spa | NULL | crawl-data/CC-MAIN-2024-22/segments/1715971057216.39/warc/CC-MAIN-20240517233122-20240518023122-00000.warc.gz | 80610731 | 17423 | 1715971057216.39 | CC-MAIN-2024-22 | warc
+
+Writing our one row to a local parquet file, whirlwind.parquet
+100% ▕████████████████████████████████████████████████████████████▏ 
+Total records for local whirlwind.parquet should be 1:
+1
+
+Our one row, locally:
+url_surtkey | url | url_host_name | url_host_tld | url_host_2nd_last_part | url_host_3rd_last_part | url_host_4th_last_part | url_host_5th_last_part | url_host_registry_suffix | url_host_registered_domain | url_host_private_suffix | url_host_private_domain | url_host_name_reversed | url_protocol | url_port | url_path | url_query | fetch_time | fetch_status | fetch_redirect | content_digest | content_mime_type | content_mime_detected | content_charset | content_languages | content_truncated | warc_filename | warc_record_offset | warc_record_length | warc_segment | crawl | subset
+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+org,wikipedia,an)/wiki/escopete | https://an.wikipedia.org/wiki/Escopete | an.wikipedia.org | org | wikipedia | an | NULL | NULL | org | wikipedia.org | org | wikipedia.org | org.wikipedia.an | https | NULL | /wiki/Escopete | NULL | 2024-05-18T01:58:10Z | 200 | NULL | RY7PLBUFQNI2FFV5FTUQK72W6SNPXLQU | text/html | text/html | UTF-8 | spa | NULL | crawl-data/CC-MAIN-2024-22/segments/1715971057216.39/warc/CC-MAIN-20240517233122-20240518023122-00000.warc.gz | 80610731 | 17423 | 1715971057216.39 | CC-MAIN-2024-22 | warc
+
+Complete row:
   url_surtkey org,wikipedia,an)/wiki/escopete
   url https://an.wikipedia.org/wiki/Escopete
   url_host_name an.wikipedia.org
   url_host_tld org
   url_host_2nd_last_part wikipedia
   url_host_3rd_last_part an
-  url_host_4th_last_part None
-  url_host_5th_last_part None
+  url_host_4th_last_part null
+  url_host_5th_last_part null
   url_host_registry_suffix org
   url_host_registered_domain wikipedia.org
   url_host_private_suffix org
   url_host_private_domain wikipedia.org
   url_host_name_reversed org.wikipedia.an
   url_protocol https
-  url_port nan
+  url_port null
   url_path /wiki/Escopete
-  url_query None
-  fetch_time 2024-05-18 01:58:10+00:00
+  url_query null
+  fetch_time 2024-05-18T01:58:10Z
   fetch_status 200
-  fetch_redirect None
+  fetch_redirect null
   content_digest RY7PLBUFQNI2FFV5FTUQK72W6SNPXLQU
   content_mime_type text/html
   content_mime_detected text/html
   content_charset UTF-8
   content_languages spa
-  content_truncated None
+  content_truncated null
   warc_filename crawl-data/CC-MAIN-2024-22/segments/1715971057216.39/warc/CC-MAIN-20240517233122-20240518023122-00000.warc.gz
   warc_record_offset 80610731
   warc_record_length 17423
   warc_segment 1715971057216.39
   crawl CC-MAIN-2024-22
   subset warc
 
-equivalent to cdxj:
-org,wikipedia,an)/wiki/escopete 20240518015810 {"url": "https://an.wikipedia.org/wiki/Escopete", "mime": "text/html", "status": "200", "digest": "sha1:RY7PLBUFQNI2FFV5FTUQK72W6SNPXLQU", "length": "17423", "offset": "80610731", "filename": "crawl-data/CC-MAIN-2024-22/segments/1715971057216.39/warc/CC-MAIN-20240517233122-20240518023122-00000.warc.gz"}
-
+Equivalent to CDXJ:
+org,wikipedia,an)/wiki/escopete 20240518015810 {"url":"https://an.wikipedia.org/wiki/Escopete","mime":"text/html","status":"200","digest":"sha1:RY7PLBUFQNI2FFV5FTUQK72W6SNPXLQU","length":"17423","offset":"80610731","filename":"crawl-data/CC-MAIN-2024-22/segments/1715971057216.39/warc/CC-MAIN-20240517233122-20240518023122-00000.warc.gz"}
 ```
 </details>
 

README.md

@@ -26,6 +26,16 @@
 			<artifactId>jwarc</artifactId>
 			<version>0.33.0</version>
 		</dependency>
+		<dependency>
+			<groupId>org.duckdb</groupId>
+			<artifactId>duckdb_jdbc</artifactId>
+			<version>1.1.3</version>
+		</dependency>
+		<dependency>
+			<groupId>com.google.code.gson</groupId>
+			<artifactId>gson</artifactId>
+			<version>2.11.0</version>
+		</dependency>
 	</dependencies>
 
 	<build>