Merge pull request #8 from midwire/feature/5-finish-refactor

midwire · web-flow · commit 1e9ec0fa5e4f · 2018-04-24T07:45:43.000-05:00
Finish refactor
diff --git a/.gitignore b/.gitignore
@@ -4,3 +4,4 @@
 /build
 /stubs
 /vendor/bundle/
+/pkg
diff --git a/.rubocop.yml b/.rubocop.yml
@@ -31,6 +31,9 @@ CommentAnnotation:
 ########################################
 # Style Cops
 
+Style/ClassVars:
+  Enabled: false
+
 Style/Documentation:
   Enabled: false
 
diff --git a/CHANGELOG b/CHANGELOG
@@ -0,0 +1,12 @@
+*1.0.1* (April 23, 2018)
+
+* Made it a gem with a command line executable bin/free_zipcode_data
+* Use Kiba for ETL
+* Support user switches for various options including custom table names and generating .csv files
+* Use in-memory SQLite database to create the tables, then save it as a file on disk
+* Separate concerns for each table
+* Add a progressbar with ETA
+* Fix a bug when looking up state_id
+* Add a ‘name’ index on states table
+* Add a switch to generate individual .csv files [--generate-files]
+* Add a switch to overwrite [--clobber] downloaded and generated .csv files
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -1,4 +1,4 @@
-## How to contribute to jsonapi_rspec
+## How to contribute to free_zipcode_data
 
 #### **Did you find a bug?**
 
diff --git a/Gemfile.lock b/Gemfile.lock
@@ -1,9 +1,10 @@
 PATH
   remote: .
   specs:
-    free_zipcode_data (1.0.0)
+    free_zipcode_data (1.0.1)
       colored (~> 1.2)
       kiba (~> 2.0)
+      ruby-progressbar (~> 1.9)
       rubyzip (~> 1.2)
       sqlite3 (~> 1.3)
       trollop (~> 2.1)
@@ -51,6 +52,7 @@ GEM
       rainbow (>= 2.2.2, < 4.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (~> 1.0, >= 1.0.1)
+    ruby-prof (0.17.0)
     ruby-progressbar (1.9.0)
     rubyzip (1.2.1)
     simplecov (0.16.1)
@@ -72,8 +74,9 @@ DEPENDENCIES
   pry-nav (~> 0.2)
   rake (~> 12.0)
   rspec (~> 3.7)
-  rubocop
-  simplecov
+  rubocop (~> 0.55)
+  ruby-prof (~> 0.17)
+  simplecov (~> 0.16)
 
 BUNDLED WITH
    1.16.1
diff --git a/Rakefile b/Rakefile
@@ -1,5 +1,10 @@
+# frozen_string_literal: true
+
 require 'rubygems'
 require 'bundler/setup'
 
 require 'rake'
 Dir['lib/tasks/**/*.rake'].sort.each { |ext| load ext }
+
+# Install rubygem tasks
+Bundler::GemHelper.install_tasks
diff --git a/free_zipcode_data.gemspec b/free_zipcode_data.gemspec
@@ -5,13 +5,17 @@ lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'free_zipcode_data/version'
 
+# rubocop:disable Metrics/BlockLength
 Gem::Specification.new do |spec|
   spec.name          = 'free_zipcode_data'
   spec.version       = FreeZipcodeData::VERSION
   spec.authors       = ['Chris Blackburn', 'Chris McKnight']
-  spec.email         = ['87a1779b@opayq.com']
-  spec.summary       = 'Free US postal codes in CSV and SQLite3 format.'
-  spec.description   = spec.summary
+  spec.email         = ['87a1779b@opayq.com', 'fixme@mcknight.bogus']
+  spec.summary       = 'Free US and world-wide postal codes in SQLite and CSV format'
+  spec.description   = <<~STRING
+    Free US and world-wide postal codes in SQLite and CSV format.
+    Automated zipcode/postal code aggregation and processing for any needs.
+  STRING
   spec.homepage      = 'https://github.com/midwire/free_zipcode_data'
   spec.license       = 'MIT'
 
@@ -25,12 +29,15 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'pry-nav', '~> 0.2'
   spec.add_development_dependency 'rake', '~> 12.0'
   spec.add_development_dependency 'rspec', '~> 3.7'
-  spec.add_development_dependency 'rubocop'
-  spec.add_development_dependency 'simplecov'
+  spec.add_development_dependency 'rubocop', '~> 0.55'
+  spec.add_development_dependency 'ruby-prof', '~> 0.17'
+  spec.add_development_dependency 'simplecov', '~> 0.16'
 
   spec.add_runtime_dependency 'colored', '~> 1.2'
   spec.add_runtime_dependency 'kiba', '~> 2.0'
+  spec.add_runtime_dependency 'ruby-progressbar', '~> 1.9'
   spec.add_runtime_dependency 'rubyzip', '~> 1.2'
   spec.add_runtime_dependency 'sqlite3', '~> 1.3'
   spec.add_runtime_dependency 'trollop', '~> 2.1'
 end
+# rubocop:enable Metrics/BlockLength
diff --git a/lib/etl/free_zipcode_data_job.rb b/lib/etl/free_zipcode_data_job.rb
@@ -30,6 +30,9 @@ def setup(country_file, database, logger, options)
           database: database,
           tablename: options[:zipcode_tablename]
 
+        post_process do
+          logger.verbose('Finished generating table data...')
+        end
       end
     end
   end
diff --git a/lib/free_zipcode_data/country_table.rb b/lib/free_zipcode_data/country_table.rb
@@ -39,6 +39,8 @@ def write(row)
       rescue SQLite3::ConstraintException
         # Swallow duplicates
       end
+
+      update_progress
     end
   end
 end
diff --git a/lib/free_zipcode_data/county_table.rb b/lib/free_zipcode_data/county_table.rb
@@ -25,8 +25,9 @@ def build
 
     def write(row)
       return nil unless row[:county]
-      state_id = get_state_id(row[:short_state])
-      raise "Could not find state: #{row[:short_state]}" unless state_id
+      state_id = get_state_id(row[:short_state], row[:state])
+      return nil unless state_id
+
       sql = <<-SQL
         INSERT INTO counties (state_id, abbr, name)
         VALUES ('#{state_id}',
@@ -42,6 +43,8 @@ def write(row)
       rescue StandardError => err
         raise "Please file an issue at #{ISSUE_URL}: [#{err}] -> SQL: [#{sql}]"
       end
+
+      update_progress
     end
   end
 end
diff --git a/lib/free_zipcode_data/db_table.rb b/lib/free_zipcode_data/db_table.rb
@@ -1,16 +1,24 @@
 # frozen_string_literal: true
 
 require 'yaml'
+require 'ruby-progressbar'
 
 module FreeZipcodeData
   class DbTable
     ISSUE_URL = 'https://github.com/midwire/free_zipcode_data/issues/new'
 
     attr_reader :database, :tablename
+    @@progressbar = nil
 
     def initialize(database:, tablename:)
       @database  = database
       @tablename = tablename
+      lc = select_first('SELECT value FROM meta where name = "line_count"')
+      @@progressbar = ProgressBar.create(total: lc.to_i * 4, format: '%t: |%B| %e')
+    end
+
+    def update_progress
+      @@progressbar.increment
     end
 
     private
@@ -19,23 +27,28 @@ def country_lookup_table
       @country_lookup_table ||= YAML.load_file('country_lookup_table.yml')
     end
 
-    def get_country_id(country)
-      rows = database.execute("SELECT id FROM countries WHERE alpha2 = '#{country}'")
+    def select_first(sql)
+      rows = database.execute(sql)
       rows[0].nil? ? nil : rows[0].first
+    rescue SQLite3::SQLException => err
+      raise "Please file an issue at #{ISSUE_URL}: [#{err}] -> SQL: [#{sql}]"
     end
 
-    def get_state_id(state)
-      rows = database.execute("SELECT id FROM states WHERE abbr = '#{state}'")
-      rows[0].nil? ? nil : rows[0].first
+    def get_country_id(country)
+      sql = "SELECT id FROM countries WHERE alpha2 = '#{country}'"
+      select_first(sql)
+    end
+
+    def get_state_id(state_abbr, state_name)
+      sql = "SELECT id FROM states
+        WHERE abbr = '#{state_abbr}' OR name = '#{escape_single_quotes(state_name)}'"
+      select_first(sql)
     end
 
     def get_county_id(county)
       return nil if county.nil?
       sql = "SELECT id FROM counties WHERE name = '#{escape_single_quotes(county)}'"
-      rows = database.execute(sql)
-      rows[0].nil? ? nil : rows[0].first
-    rescue SQLite3::SQLException => err
-      raise "Please file an issue at #{ISSUE_URL}: [#{err}] -> SQL: [#{sql}]"
+      select_first(sql)
     end
 
     def escape_single_quotes(string)
diff --git a/lib/free_zipcode_data/runner.rb b/lib/free_zipcode_data/runner.rb
@@ -6,9 +6,10 @@
 
 require_relative '../etl/free_zipcode_data_job'
 
-require 'pry' if ENV.fetch('APP_ENV') == 'development'
+require 'pry' if ENV.fetch('APP_ENV', '') == 'development'
 
 module FreeZipcodeData
+  # rubocop:disable Metrics/ClassLength
   class Runner
     attr_accessor :logger, :options
 
@@ -23,30 +24,38 @@ def initialize
 
     def start
       start_time = Time.now
-      options = FreeZipcodeData::Options.instance
-      options.initialize_hash(collect_args)
+      opt = FreeZipcodeData::Options.instance
+      opt.initialize_hash(collect_args)
+      @options = opt.hash
 
-      logger.info('Starting FreeZipcodeData...'.green)
+      logger.info("Starting FreeZipcodeData v#{VERSION}...".green)
 
-      datasource = DataSource.new(options.hash.country)
+      datasource = DataSource.new(options.country)
       datasource.download
 
-      database = SqliteRam.new(File.join(options.hash.work_dir, 'free_zipcode_data.sqlite3'))
+      db_file = File.join(options.work_dir, 'free_zipcode_data.sqlite3')
+      database = SqliteRam.new(db_file)
+      configure_meta(database.conn, datasource.datafile)
 
       %i[country state county zipcode].each { |t| initialize_table(t, database) }
 
       extract_transform_load(datasource, database)
 
+      logger.info("Saving database to disk '#{db_file}'...")
       database.save_to_disk
 
-      elapsed = Time.now - start_time
-      logger.info("Finished in [#{elapsed}] seconds.".yellow)
+      if options.generate_files
+        logger.info('Generating .csv files...')
+        database.dump_tables(options.work_dir)
+      end
+
+      elapsed = Time.at(Time.now - start_time).utc.strftime('%H:%M:%S')
+      logger.info("Processed #{datasource_line_count} zipcodes in [#{elapsed}].".yellow)
     end
 
     private
 
     def initialize_table(table_sym, database)
-      options = Options.instance.hash
       tablename = options["#{table_sym}_tablename".to_sym]
       logger.verbose("Initializing #{table_sym} table: '#{tablename}'...")
       klass = instance_eval("#{titleize(table_sym)}Table", __FILE__, __LINE__)
@@ -57,12 +66,37 @@ def initialize_table(table_sym, database)
       table.build
     end
 
+    def datasource_line_count(filename)
+      @datasource_line_count ||= begin
+        count = File.foreach(filename).inject(0) { |c, _line| c + 1 }
+        logger.verbose("Processing #{count} zipcodes in '#{filename}'...")
+        count
+      end
+    end
+
+    def configure_meta(database, datasource)
+      schema = <<-SQL
+        create table meta (
+          id integer not null primary key,
+          name varchar(255),
+          value varchar(255)
+        )
+      SQL
+      database.execute_batch(schema)
+
+      sql = <<-SQL
+        INSERT INTO meta (name, value)
+        VALUES ('line_count', #{datasource_line_count(datasource)})
+      SQL
+      database.execute(sql)
+    end
+
     def extract_transform_load(datasource, database)
       job = ETL::FreeZipcodeDataJob.setup(
         datasource.datafile,
         database.conn,
         logger,
-        FreeZipcodeData::Options.instance.hash
+        options
       )
       Kiba.run(job)
     end
@@ -71,15 +105,20 @@ def extract_transform_load(datasource, database)
     # rubocop:disable Metrics/MethodLength
     def collect_args
       Trollop.options do
+        opt(
+          :work_dir,
+          'REQUIRED: Specify your work/build directory, where the SQLite and .csv files will be built',
+          type: :string, required: true, short: '-w'
+        )
         opt(
           :country,
           'Specify the country code for processing, or all countries if not specified',
-          type: :string, required: false, short: '-g'
+          type: :string, required: false, short: '-f'
         )
         opt(
-          :work_dir,
-          'Specify your work/build directory, where the SQLite and .csv files will be built',
-          type: :string, required: true, short: '-w'
+          :generate_files,
+          'Generate CSV files: [counties.csv, states.csv, countries.csv, zipcodes.csv]',
+          type: :boolean, required: false, short: '-g', default: false
         )
         opt(
           :country_tablename,
@@ -129,4 +168,5 @@ def titleize(string)
       ret
     end
   end
+  # rubocop:enable Metrics/ClassLength
 end
diff --git a/lib/free_zipcode_data/sqlite_ram.rb b/lib/free_zipcode_data/sqlite_ram.rb
@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'sqlite3'
+require 'csv'
 
 # Open a SQlite DB, work with it in-memory and save back to disk
 class SqliteRam
@@ -18,4 +19,21 @@ def save_to_disk
     backup.step(-1)
     backup.finish
   end
+
+  def dump_tables(path)
+    tables = conn.execute('select name from sqlite_master where type = "table"')
+    sql = nil
+    tables.each do |table_array|
+      table = table_array.first
+      headers_sql = "pragma table_info('#{table}')"
+      header = conn.execute(headers_sql).map { |e| e[1] }
+      CSV.open(File.join(path, "#{table}.csv"), 'w') do |csv|
+        csv << header
+        sql = "select * from #{table}"
+        conn.execute(sql).each do |row_array|
+          csv << row_array
+        end
+      end
+    end
+  end
 end
diff --git a/lib/free_zipcode_data/state_table.rb b/lib/free_zipcode_data/state_table.rb
@@ -20,6 +20,12 @@ def build
         ON #{tablename} (abbr, country_id COLLATE NOCASE ASC);
       SQL
       database.execute_batch(ndx)
+
+      ndx = <<-SQL
+        CREATE UNIQUE INDEX "main"."state_name"
+        ON #{tablename} (name COLLATE NOCASE ASC);
+      SQL
+      database.execute_batch(ndx)
     end
 
     def write(row)
@@ -38,6 +44,8 @@ def write(row)
       rescue SQLite3::ConstraintException
         # Swallow duplicates
       end
+
+      update_progress
     end
   end
 end
diff --git a/lib/free_zipcode_data/version.rb b/lib/free_zipcode_data/version.rb
diff --git a/lib/free_zipcode_data/zipcode_table.rb b/lib/free_zipcode_data/zipcode_table.rb
diff --git a/lib/tasks/version.rake b/lib/tasks/version.rake

-Original file line number
+Diff line change
 /build
 /stubs
 /vendor/bundle/
 +/pkg
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-## How to contribute to jsonapi_rspec`
	`1`	`+## How to contribute to free_zipcode_data`
`2`	`2`
`3`	`3`	`#### Did you find a bug?`
`4`	`4`