Merge branch 'master' into feat/add-microsoft.com

Author: gildesmarais

Gemfile

@@ -11,6 +11,7 @@ group :development do
   gem 'html2rss-generator', github: 'html2rss/generator', branch: :main
 
   gem 'nokogiri'
+  gem 'public_suffix'
   gem 'rspec', '~> 3.0'
   gem 'rubocop'
   gem 'rubocop-performance'

Gemfile.lock

@@ -12,7 +12,7 @@ GIT
 
 GIT
   remote: https://github.com/html2rss/html2rss
-  revision: 861cc9cae785510cc7120185b458390c18fd615a
+  revision: 400e796540e82a69e1f1e014b6f89c626acf32fd
   branch: master
   specs:
     html2rss (0.17.0)
@@ -44,13 +44,13 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    addressable (2.8.7)
-      public_suffix (>= 2.0.2, < 7.0)
+    addressable (2.8.8)
+      public_suffix (>= 2.0.2, < 8.0)
     ast (2.4.3)
     base64 (0.3.0)
     bigdecimal (3.3.1)
     brotli (0.7.0)
-    concurrent-ruby (1.3.5)
+    concurrent-ruby (1.3.6)
     crass (1.0.6)
     diff-lcs (1.6.2)
     dry-configurable (1.3.0)
@@ -92,16 +92,16 @@ GEM
       faraday-net_http (>= 2.0, < 3.5)
       json
       logger
-    faraday-follow_redirects (0.4.0)
+    faraday-follow_redirects (0.5.0)
       faraday (>= 1, < 3)
     faraday-gzip (3.0.4)
       faraday (>= 2.0, < 3)
       zlib (~> 3.0)
-    faraday-net_http (3.4.1)
-      net-http (>= 0.5.0)
+    faraday-net_http (3.4.2)
+      net-http (~> 0.5)
     hashie (5.0.0)
     htmlbeautifier (1.4.3)
-    json (2.15.2)
+    json (2.18.0)
     kramdown (2.5.1)
       rexml (>= 3.3.9)
     language_server-protocol (3.17.0.5)
@@ -111,8 +111,12 @@ GEM
       logger
       mime-types-data (~> 3.2025, >= 3.2025.0507)
     mime-types-data (3.2025.0924)
-    net-http (0.7.0)
-      uri
+    mini_portile2 (2.8.9)
+    net-http (0.9.1)
+      uri (>= 0.11.1)
+    nokogiri (1.18.8)
+      mini_portile2 (~> 2.8.2)
+      racc (~> 1.4)
     nokogiri (1.18.8-arm64-darwin)
       racc (~> 1.4)
     nokogiri (1.18.8-x86_64-darwin)
@@ -134,7 +138,7 @@ GEM
     racc (1.8.1)
     rainbow (3.1.1)
     regexp_parser (2.11.3)
-    reverse_markdown (3.0.0)
+    reverse_markdown (3.0.1)
       nokogiri
     rexml (3.4.4)
     rouge (4.5.2)
@@ -151,7 +155,7 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
     rspec-support (3.13.4)
-    rss (0.3.1)
+    rss (0.3.2)
       rexml
     rubocop (1.75.8)
       json (~> 2.3)
@@ -205,7 +209,7 @@ GEM
       concurrent-ruby (~> 1.0)
     unicode-display_width (2.6.0)
     unicode_utils (1.4.0)
-    uri (1.1.0)
+    uri (1.1.1)
     websocket-driver (0.8.0)
       base64
       websocket-extensions (>= 0.1.0)
@@ -225,6 +229,7 @@ DEPENDENCIES
   html2rss-configs!
   html2rss-generator!
   nokogiri
+  public_suffix
   rspec (~> 3.0)
   rubocop
   rubocop-performance

README.md

@@ -57,6 +57,8 @@ make test-domain DOMAIN=github.com
 
 **Adding new configs**: Just create the YAML file and run tests. No spec file needed.
 
+**Config folder convention**: Place configs under the registrable domain folder (e.g., `example.com/` or `bbc.co.uk/`). Legacy subdomain folders (e.g., `news.example.com/`) are allowed but not preferred.
+
 ## Documentation
 
 - [Main Documentation](https://html2rss.github.io/html2rss-configs/)

spec/helper_spec.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+RSpec.describe Helper do
+  describe '.url_to_registrable_domain' do
+    it 'collapses subdomains to the registrable domain' do
+      expect(described_class.url_to_registrable_domain('https://blog.example.com/posts')).to eq('example.com')
+    end
+
+    it 'keeps multi-part TLDs intact for registrable domain' do
+      expect(described_class.url_to_registrable_domain('https://news.bbc.co.uk/world')).to eq('bbc.co.uk')
+    end
+
+    it 'preserves single-host domains' do
+      expect(described_class.url_to_registrable_domain('https://example.com')).to eq('example.com')
+    end
+
+    it 'returns nil for blank or invalid URLs', :aggregate_failures do
+      expect(described_class.url_to_registrable_domain(nil)).to be_nil
+      expect(described_class.url_to_registrable_domain('')).to be_nil
+      expect(described_class.url_to_registrable_domain('not a url')).to be_nil
+    end
+  end
+
+  describe '.url_to_host_name' do
+    it 'returns the full host' do
+      expect(described_class.url_to_host_name('https://news.bbc.co.uk/world')).to eq('news.bbc.co.uk')
+    end
+
+    it 'returns nil for blank or invalid URLs', :aggregate_failures do
+      expect(described_class.url_to_host_name(nil)).to be_nil
+      expect(described_class.url_to_host_name('')).to be_nil
+      expect(described_class.url_to_host_name('not a url')).to be_nil
+    end
+  end
+
+  describe 'legacy naming guardrail' do
+    it 'does not expose url_to_directory_name' do
+      expect(described_class).not_to respond_to(:url_to_directory_name)
+    end
+  end
+
+  describe '.registrable_domain' do
+    it 'falls back to host when PublicSuffix returns nil' do
+      allow(PublicSuffix).to receive(:domain).with('example.local').and_return(nil)
+
+      expect(described_class.send(:registrable_domain, 'example.local')).to eq('example.local')
+    end
+
+    it 'falls back to host when PublicSuffix raises DomainInvalid' do
+      allow(PublicSuffix).to receive(:domain).with('invalid..host')
+                                             .and_raise(PublicSuffix::DomainInvalid)
+
+      expect(described_class.send(:registrable_domain, 'invalid..host')).to eq('invalid..host')
+    end
+  end
+end

spec/support/helper.rb

@@ -3,16 +3,37 @@
 require 'json'
 require 'nokogiri'
 require 'yaml'
-require 'uri'
+require 'public_suffix'
 
 ##
 # A collection of helper methods.
 module Helper
   ##
   # @param url [String]
+  # @return [String, nil]
+  def self.url_to_registrable_domain(url)
+    host = url_to_host_name(url)
+    return host unless host
+
+    registrable_domain(host)
+  end
+
+  ##
+  # @param url [String]
+  # @return [String, nil]
+  def self.url_to_host_name(url)
+    Html2rss::Url.for_channel(url)&.host
+  rescue ArgumentError
+    nil
+  end
+
+  ##
+  # @param host [String]
   # @return [String]
-  def self.url_to_directory_name(url)
-    URI(url.split('/')[0..2].join('/')).host.gsub(/^(api|www|webapp)\./, '')
+  def self.registrable_domain(host)
+    PublicSuffix.domain(host) || host
+  rescue PublicSuffix::DomainInvalid
+    host
   end
 
   ##

spec/support/shared_examples/config.yml_spec.rb

@@ -41,16 +41,17 @@
     config
   end
 
-  context 'with the file' do
-    let(:host_name) { Helper.url_to_directory_name yaml['channel']['url'] }
+  context 'with the file' do # rubocop:disable RSpec/MultipleMemoizedHelpers
+    let(:host_name) { Helper.url_to_host_name yaml['channel']['url'] }
+    let(:domain_name) { Helper.url_to_registrable_domain yaml['channel']['url'] }
     let(:dirname) { File.dirname(file_path).split(File::Separator).last }
 
     it 'is parseable' do
       expect { yaml }.not_to raise_error
     end
 
-    it "resides in a folder named after channel.url's host" do
-      expect(dirname).to eq(host_name)
+    it "resides in a folder named after channel.url's host or domain" do
+      expect([domain_name, host_name]).to include(dirname)
     end
   end