public-apis/build/validate_links.rb

#!/usr/bin/env ruby
require 'httparty'
require 'ruby-progressbar'
require 'uri'
allowed_codes = [200, 302, 403, 429]
allowed_links = ["https://www.yelp.com/developers/documentation/v3"]
args = ARGV
filename = args[0]
contents = File.open(filename, 'rb') { |f| f.read }
raw_links = URI.extract(contents, ['http', 'https'])
# Remove trailing ')' from entry URLs
links = []
raw_links.each do |link|
    if link.end_with?(')')
        links.push(link[0...-1])
    else
        links.push(link)
    end
end
if links.length <= 0
    puts "no links to check"
    exit(0)
end
fails = []
# Fail on any duplicate elements
dup = links.select{|element| links.count(element) > 1}
if dup.uniq.length > 0
    dup.uniq.each do |e|
        fails.push("(DUP): #{e}")
    end
end
# Remove any duplicates from array
links = links.uniq
count = 0
total = links.length
progressbar = ProgressBar.create(:total => total,
    :format => "%a %P% | Processed: %c from %C")
# GET each link and check for valid response code from allowed_codes
links.each do |link|
    begin
        count += 1
        if allowed_links.include?(link)
            next
        end
        res = HTTParty.get(link, timeout: 10)
        if res.code.nil?
            fails.push("(NIL): #{link}")
            next
        end
        if !allowed_codes.include?(res.code)
            fails.push("(#{res.code}): #{link}")
        end
    rescue Net::ReadTimeout
        fails.push("(TMO): #{link}")
    rescue Net::OpenTimeout
        fails.push("(TMO): #{link}")
    rescue OpenSSL::SSL::SSLError
        fails.push("(SSL): #{link}")
    rescue SocketError
        fails.push("(SOK): #{link}")
    rescue Errno::ECONNREFUSED
        fails.push("(CON): #{link}")
    end
    progressbar.increment
end
puts "#{count}/#{total} links checked"
if fails.length <= 0
    puts "all links valid"
    exit(0)
else
    puts "-- RESULTS --"
    fails.sort!
    fails.each do |e|
        puts e
    end
    exit(1)
end