The Background Affiliation Status Information Center (BASIC) on the National Futures Association website includes detailed membership information on those in the “U.S. derivatives industry, including on-exchange traded futures, retail off-exchange foreign currency (forex) and OTC derivatives (swaps).”. This includes information on those individuals and entities that have been found in non-compliance with regulations.
The listing here was generated from a “crawl” through the 28,785 membership listings on the site
, and includes only those (1,420 entities) with current regulatory compliance findings against them, including details on these findings.
The Ruby script used to “crawl” the site and extract the data is as follows:
require ‘rubygems’
require ‘watir-webdriver’
ary_of_links = Array.new
rows = Array.new
### Defines Array Class for HTML Table output ###
class Array
def to_cells(tag)
self.map { |c| “<#{tag}>#{c}</#{tag}>” }.join
end
end
browser = Watir::Browser.new :chrome
[*(‘A’..’Z’)
antibiotika-online.com
,*(‘0’..’9′)].each do |letter|
browser.goto ‘http://www.nfa.futures.org/basicnet/welcome.aspx’
browser.text_field(:name, ‘_ctl0:cphMain:txtFirm’).set letter
browser.link(:onclick=>”SubmitSearch(‘FIRM’);”).click
loop do
table = browser.table(:id, ‘_ctl0_cphMain_grdType1’)
### First Page ###
browser.table(:id, ‘_ctl0_cphMain_grdType1’)[0][0].links.each do |linkz|
if linkz.text.to_s !~ /…/
ary_of_links << linkz.href
end
end
tbl_Rows = browser.table(:id, ‘_ctl0_cphMain_grdType1’).rows.length
for t in 2..tbl_Rows.to_i-2
registrant = browser.table(:id, ‘_ctl0_cphMain_grdType1’)[t][0].text
browser.table(:id, ‘_ctl0_cphMain_grdType1’)[t][0].link(:index, 0).click
details_tbl_Rows = browser.table(:xpath, ‘//*[@id=”_ctl0_cphMain_tdCases1″]/table/tbody/tr[2]/td/table/tbody/tr/td/table’).rows.length
for i in 1..details_tbl_Rows.to_i-2
if browser.table(:xpath, ‘//*[@id=”_ctl0_cphMain_tdCases1″]/table/tbody/tr[2]/td/table/tbody/tr/td/table’)[i][1].text.to_i > 0
registrant =”#{registrant}”
browser.link(:text, ‘details…’).click
reg_details_Rows = browser.table(:xpath, ‘//*[@id=”_ctl0_cphMain_grdTypeReg”]’).rows.length
for p in 1..reg_details_Rows.to_i-1
source = browser.table(:xpath, ‘//*[@id=”_ctl0_cphMain_grdTypeReg”]’)[p][0].text
case_link = browser.table(:xpath, ‘//*[@id=”_ctl0_cphMain_grdTypeReg”]’)[p][1].link.href
case_nbr = browser.table(:xpath, ‘//*[@id=”_ctl0_cphMain_grdTypeReg”]’)[p][1].text
case_detail = “<a href=\”#{case_link}\”>#{case_nbr}</a>”
action_type = browser.table(:xpath, ‘//*[@id=”_ctl0_cphMain_grdTypeReg”]’)[p][2].html
action_type.gsub!(/•/,’-‘)
action_type.gsub!(/<td style=”font-size:9px;width:190px;”>|<\/td>|<td style=”font-size:9px;”>/,”)
outcomes = browser.table(:xpath, ‘//*[@id=”_ctl0_cphMain_grdTypeReg”]’)[p][4].html
outcomes.gsub!(/•/,’-‘)
outcomes.gsub!(/<td style=”font-size:9px;width:190px;”>|<\/td>|<td style=”font-size:9px;”>/,”)
rows << {“Name” => registrant, “Contributor” => source, “Case #” => case_detail, “Action Type” => action_type, “Outcomes” => outcomes}
end
browser.back
break
end
end
browser.back
end
### Subsequent Pages ###
if browser.table(:id, ‘_ctl0_cphMain_grdType1’)[0][0].text !~ /\.\.\..*?\d+\Z/
ary_of_links.each do |item|
browser.link(:href => “#{item}”).click
tbl_Rows = browser.table(:id, ‘_ctl0_cphMain_grdType1’).rows.length
for t in 2..tbl_Rows.to_i-2
registrant = browser.table(:id, ‘_ctl0_cphMain_grdType1’)[t][0].text
puts “#{t}\t#{registrant}”
browser.table(:id, ‘_ctl0_cphMain_grdType1’)[t][0].link(:index, 0).click
details_tbl_Rows = browser.table(:xpath, ‘//*[@id=”_ctl0_cphMain_tdCases1″]/table/tbody/tr[2]/td/table/tbody/tr/td/table’).rows.length
for i in 1..details_tbl_Rows.to_i-2
if browser.table(:xpath, ‘//*[@id=”_ctl0_cphMain_tdCases1″]/table/tbody/tr[2]/td/table/tbody/tr/td/table’)[i][1].text.to_i > 0
browser.link(:text, ‘details…’).click
reg_details_Rows = browser.table(:xpath, ‘//*[@id=”_ctl0_cphMain_grdTypeReg”]’).rows.length
for p in 1..reg_details_Rows.to_i-1
source = browser.table(:xpath, ‘//*[@id=”_ctl0_cphMain_grdTypeReg”]’)[p][0].text
case_link = browser.table(:xpath, ‘//*[@id=”_ctl0_cphMain_grdTypeReg”]’)[p][1].link.href
case_nbr = browser.table(:xpath, ‘//*[@id=”_ctl0_cphMain_grdTypeReg”]’)[p][1].text
case_detail = “<a href=\”#{case_link}\”>#{case_nbr}</a>”
action_type = browser.table(:xpath, ‘//*[@id=”_ctl0_cphMain_grdTypeReg”]’)[p][2].html
action_type.gsub!(/•/,’-‘)
action_type.gsub!(/<td style=”font-size:9px;width:190px;”>|<\/td>|<td style=”font-size:9px;”>/,”)
outcomes = browser.table(:xpath, ‘//*[@id=”_ctl0_cphMain_grdTypeReg”]’)[p][4].html
outcomes.gsub!(/•/,’-‘)
outcomes.gsub!(/<td style=”font-size:9px;width:190px;”>|<\/td>|<td style=”font-size:9px;”>/,”)
rows << {“Name” => registrant, “Contributor” => source, “Case #” => case_detail, “Action Type” => action_type, “Outcomes” => outcomes}
end
browser.back
break
end
end
browser.back
end
end
end
ary_of_links.clear
### Upon reaching the end of the “Subsequent Pages” the next section handles new table ranges, or if at the end of a query exists the loop ###
if browser.table(:id, ‘_ctl0_cphMain_grdType1′)[0][0].text =~ /\.\.\..*?\.\.\./
browser.link(:text =>’…’, index: 1).click
elsif browser.table(:id, ‘_ctl0_cphMain_grdType1’)[0][0].text =~ /.*?\d+\s+\.\.\./
browser.link(:text =>’…’).click
else
break
end
end
end
### Rolls HTML Table output ###
headers = “<tr>#{rows[0].keys.to_cells(‘th’)}</tr>”
cells = rows.map do |row|
“<tr>#{row.values.to_cells(‘td’)}</tr>”
end.join(“\n “)
table = “<table border=\”1\”>
#{headers}
#{cells}
</table>”
puts table