From 00f734009cb0fee6513764b362a1517f707dd247 Mon Sep 17 00:00:00 2001 From: Cam Peterson Date: Fri, 22 Nov 2013 12:15:18 -0700 Subject: [PATCH] Refactor ruby gem, adding twofishes api call and update version # --- rubygem/Gemfile.lock | 11 ++-- rubygem/dstk-0.50.3.gem | Bin 4608 -> 0 bytes rubygem/dstk-0.51.0.gem | Bin 0 -> 5632 bytes rubygem/dstk.gemspec | 2 +- rubygem/lib/dstk.rb | 132 +++++++++++++++++--------------------- rubygem/test/test_dstk.rb | 7 ++ 6 files changed, 71 insertions(+), 81 deletions(-) delete mode 100644 rubygem/dstk-0.50.3.gem create mode 100644 rubygem/dstk-0.51.0.gem diff --git a/rubygem/Gemfile.lock b/rubygem/Gemfile.lock index 9a41d99..904694a 100644 --- a/rubygem/Gemfile.lock +++ b/rubygem/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - dstk (0.50.2) + dstk (0.51.0) httmultiparty httparty json @@ -12,12 +12,11 @@ GEM httmultiparty (0.3.10) httparty (>= 0.7.3) multipart-post - httparty (0.11.0) - multi_json (~> 1.0) + httparty (0.12.0) + json (~> 1.8) multi_xml (>= 0.5.2) - json (1.8.0) - multi_json (1.7.6) - multi_xml (0.5.4) + json (1.8.1) + multi_xml (0.5.5) multipart-post (1.2.0) PLATFORMS diff --git a/rubygem/dstk-0.50.3.gem b/rubygem/dstk-0.50.3.gem deleted file mode 100644 index f39287cf622a5964b2814c04405d22037e273adf..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4608 zcmeHJXH*kP7A7DiAiakUfdE0IgeoW~RcfO25~5&22}m^*K?4?=l%Rk}?+`*0P!L5B zr3DZHA&5LsTBuS)k+Qxm=Xl&7d*0jg&h8JtKX+#C+?l!G%(>sVqk>S1L8t&lbQsNF z1*Pv$hrwumx8L!zRnbsZ|JT?*{K~4TDrz+1N`KXXeq1*=@O$>(x4(k%@bLN3@l*Rh z`Tw5U56S%${{Pw)i)W&th7ZAM=$7rn{U+2o)}{`4(iyVdhf`J_ir5x<-VXtcVlmO+ zm72$ydS(yFMto{1UR3Ra)e}_xlzYiCf^k?qrWagEJ6|TFjpaGBw$4Zlv<3yNIVA$_ ztcjiK&(&1V>6l*sR8zpaHC<@j0oid|Cvyl&aiAcM5!_MsQ$=pkg8 zUO^Wo#xb5dQ>}ZGQBk;vjolR?97F$LI91?@7=2Q9ScO~ac}47yWIPzUm`%@K&$O?r zwO>rfVPJ6Ei1}XQ_xeLxuiLovKn!R{4TWx4-ed? zMdHS`p@=nR*WJMIxT8OjP6`9tCaN{HpzL3{39;$*h00%<1GAy@XCK`1%6)R;fcG)2 zC_-G^y=Nv9y~3vb3BzLo>a#K3*&(z-PAxR|IP;Vz%v5S{kH1(+Is~6UvV|N7EbDBa z!ik0S@A~uGrq6I0;Q2ld2aD?9l*aa#xAYPi(Jn*=7Sd2viab}3=#$Kv>-8wO;Idrl%j;IGWhUiZ#=XHIugZJQqcjO6Sv6w2<1spfKpY!gRsEdoP?0mmZdb$L zlw{=Ful?mq4FTKJu4hc?&Ym^I8yd%(jDBhCOZFl&>{ykS8dr#fs>)9e6?}$q4ZlMk zTP)wWxXd39O%lGZ9Gmm;DObgn9M5?Wvty%AixrTcQZ7n#gH2B(f*~7=*;?eAZ^BO%9ieQ&BdDlG zFW=K>1$6>KX)j{qbKW6Ezy%*ilDM<$|fO|(ePAw<7n7k@IOj5)@i$Op$D8z=yhMpk|BGUokW z7}qG|q>dnLqQ~gPH-(&lNur6kDwiNg{(0@WA(+Vs47-YEWvbG;wIh8>rW27DpfHLX z=vDG1n9r&CK@}V_220fIQ8!|xTLAN(T^A(FFFQy!8}(fFnrAs)rQB0Sz$PTse}4Lo zDbM3bB|JOsX8GBNKz5cygr`0jb$bu*sHav-PjWq%xBc<{U>42UFRTKr5azlN52Mh? z_Ivo|oW5RK&Dn}V(CM$PuDYa{Saj>chbKtK#d+hq{=8qq%2@fYaEnd{>u`|t-whAq z3oHx;Nl+fOe3M;=RT<7mS)g4jvg4ry!tp%Ma=2k-z*FC#I9)$G;9~_#mGY9Nx)zXU z=v@O{_*DnpYJjwzyb_lBX7-ZAAfhRu-C40fl9{>!vTe}wv?lxcSexW1`nfBoKPv8) zM5jSGTOC4YC7X`=ko4<1SvNDy_P7ZZ;u&554aGud>b(-Or8qbRw=#u+d~qpW>}=>+ znBPv_e(gkXqC6?~bsp#^wrm=;RE=-k@fMIhi3qc_YeFuq`bK?l*-ik))<|NRf}ztewZ6=l_b z(|-;1U;6)N?(>hv{Gk7~7Jd^-M_ZTdYD!(bUSZ)rN;N2Y=A$jVOEmx?hBGOGPKnZ? zJgsVnk!h(g3#Xu#qf7agzuEJ2ze<8=8^YTF7Mmjqebuio?~A&)AQLgm?I5x`ZnmKV zMCmfy=9Y^qi+nM+8D<7}CvP+BV3QN)4h9Qb8-FT%slO?iTq*I@I&9{oHPHr;-o=E3 zH7pP`ISw)uq_ZAxR~3Uk+6jVv&8w@#BfgOw$p!uhkzBr!$^z}{6&0IJ#76!In`3tt zI;(3l@;tl7@B+-;xE~+LKB_u&TIdK{n=an`gowxO&}iZInupE|RB@AMzwK?%b-CSOiw0J0 zi!3xsCvbj9X`5yxs6Y6~Rp*fzlXsPcGq6FY1CNjHHZJL2HyERhe+?gH##_4K1E0so zzPaH3!kfa~CM;m$(`zq3-f)JsFk$n`;%8{m79xT7eO{*h0Ha`sXIOmHmfYcl^x{p5 z+x;7pRAO#SS#h=#0f9s|Qyp?;vE|p1n+^@St%Px!FBpJLLiK zyp=NFr4R(1d7QrNmV{V(buU4c7dwBWh|FbIz_)h5JHD7xp{Z4BdR~!hlQfE^Jz2$0 xpYgmzN3eMMgv_?~rGVI}secwPi=zgluS|dR;r^Qci-BJZ{9@o21OF8Re*@6~Hr413fzYKGB1NP}Y0?sDp%Xevi3kK0r3lhQf=DM2k)2&^Eo?t5p??(xTazkF}cS?`}abMBcr_kJ_;eREL=KYJ&8KYOWL!GIrS zGT#gg1_OTg-|TxYD+^Ho$bvv%Ie7&IIS>ROBP$1%mj?*Q{HTQfy05>lpS{nwlmeU) z2=~7Rz8n9y_`hrWv$*e;|6Qm2@j$@v%6*e#)T`#ko&z#0;d^C@S-bc=3P2EgOxA^U z7Ucju*UMJZP1xii&xA&p-#h9ab%A%}M?CDcO|d}vd|g|s?OD_}X4{BvZ9#Hyz@1-* zN0pXJ(}K}12}s_$tucEh)Er?yj;+OPYG&Ks(gXm>DGXJY7MK5&o*H4z)UHcOtktNn zIG!Oa96C6ddu>7ES47ph_CeDAdlROuGDcQN{(yy6c!OC9QZvJB`vr6-x4vd%__!$3rH6-Xxynk0UKwS;A077A>DrI`Bv3=w(EK8k+~ye z*PS2p6;6`W(MC_ZEAH5-2}=?*rvc-FZ8ZJHwDFKO!B1}Q5IO8LqYR2_w;4byb{Jyx z`+fGMSLd;(K2+p2yInvjj$+CYX@)sr45um;vg~z<#3kVY=fHFB5iJzO=aEhqWr#J? zEiEqIAXfPjU6KJ~G+8-C&jNL`SwMK#fB$pJTX)fJ`m#Q3F=sut6>NN7ec$Y;OPF$Z z%q|@_VV|$!LYkC9Qf!R+wfu=QIdJl0n;xX@68kStK0U!bWi+pse`LqoF1UQT>hAib zK^7s1Md47`lwzVN%v33;%}PcI6;Q(s4p8AyaU|zeb*ZiAH)nmWcRaa2-!tLUD}I8_ zgvhMt7~=fMc2)xQnzcn_jwQ##R8@3wKD0E~CWz98OyG@Zxq0(z$ch|0Y!OU1CqWo( z@dHdB`WsJZ{nYn*T1;`8w36y`5y#KgwHe>QR5I zLB-6bBfKCaLggB{!C=ZLWzBPL6vy(F9q#25{;Hmu^PY^c*vc!*%F)hEpy=rwJ+J%s zS|yn;J(t$Ps?O+f#R65$um_L8bs5b{V$qkb&)N4zKU9|fG}Q^@gBw&=3x8qT1`^|D zXF{>2E>!$Pa)9&^U)`m!5bxaudDRncJweFia(*1JIJaoRDKQ-m9TcTBtO4+e(ULsIS|G#DU(K|bEDJMuB_ z=ET{Q+s5evk9m}OO-A>0KG?hv0IJk;9LwS!qCM+WdC}mc^TtSiBUYNRu5*OJCWQ7l ztei`SHhJkm2K_~}_S24AX07{Vc72s0+I6WO+f19ROxr=2pFR>mt#m4JEVH~RO0+O` zqxFhwP0`M{V0zPto&ug$a9V+;!4Sa9{3w4qNT!ETY2?uR%np3aLmk3k|o>QpP8| z=N1*@lLxEm_~=bi(yqFLA z{2Oj%UPua|R+F33-=R`e)`Q6N!FA4@r)@=73zdd+vK#JbwO;Oi_?qqx^wd+8F+`~Z zU4+rqCwTD=tK&jV<6(?Wd^1f=sFDlY#*Hf0Cr2v6a!>HirO6IpvcfrPswH-=G$f{k{7NL@{-!&DMGZ$ZA^XVKQSC8@Udc;A=Lq`gsrj@BG!xOOJAK7)p! zy7AQf6qq-Vt2wYsc{#I$H){)C!B@rPvT9P7> zj`$6)iVkM)&L|;^B)lyB=-nDIWVo)jLd7i{s3<#m(D=+@C=r8WpjkInErCKc zNoQ3}i;a?GvKeEBR6vXk2$<-SeWj@KWq$6Q=%bR$iLN+K_}>x%uI&55lc*DlnX@ z|B(m5B-S~++LoGZ=8bm}(c*_(w~Qhu(aS+MDHl|EMgnint|aKU8(?2&j3=Mw7E_|{%o>thJA?9<;j%DJJs| z7)=ylKJr~|)~MINdpA$*6==G#9t%`+nBjZVDu>DE>vz(n&*)a)W$CNv4C2g*BakQY z@$TZIVl#ZTej6jLfC`HwJk3Ybv&CBxk;O?%PbA7kXFD7ByvP#QnuS3an_<3^1@^WN z8A~tnP|ig!Emh$oy(x)&OAXu@3R{M%vMY)9ZARv~0-02pzwc3e1ym?zqZ_#;u~!I9 ay_p-ut(5!w^Z02$5%`I~PXzu!2>b>#AO?g0 literal 0 HcmV?d00001 diff --git a/rubygem/dstk.gemspec b/rubygem/dstk.gemspec index c1c91e2..69b8728 100644 --- a/rubygem/dstk.gemspec +++ b/rubygem/dstk.gemspec @@ -1,6 +1,6 @@ Gem::Specification.new do |s| s.name = 'dstk' - s.version = '0.50.3' + s.version = '0.51.0' s.date = '2013-07-02' s.summary = "Data Science Toolkit client" s.description = "An interface to the datasciencetoolkit.org open API for geocoding addresses, extracting entities and sentiment from unstructured text, and other common semantic and geo data tasks." diff --git a/rubygem/lib/dstk.rb b/rubygem/lib/dstk.rb index 8eecbbe..09ef190 100644 --- a/rubygem/lib/dstk.rb +++ b/rubygem/lib/dstk.rb @@ -10,142 +10,122 @@ module DSTK class DSTK + def initialize(options = {}) default_options = { :api_base => 'http://www.datasciencetoolkit.org', :check_version => true, } - if ENV['DSTK_API_BASE'] - default_options[:api_base] = ENV['DSTK_API_BASE'] - end + default_options[:api_base] = ENV['DSTK_API_BASE'] if ENV['DSTK_API_BASE'] default_options.each do |key, value| - if !options.has_key?(key) - options[key] = value - end + options[key] = value if !options.has_key?(key) end - - @api_base = options[:api_base] - if options[:check_version] - self.check_version() - end + @dstk_api_base = options[:api_base] + + self.check_version() if options[:check_version] end - # A short-hand method to URL encode a string. See http://web.elctech.com/?p=58 + ############### UTILITIES ################### + # a short-hand method to URL encode a string. See http://web.elctech.com/?p=58 def u(str) str.gsub(/[^a-zA-Z0-9_\.\-]/n) {|s| sprintf('%%%02x', s[0]) } end - def json_api_call(endpoint, arguments = {}, data_payload = nil, data_payload_type = 'json') + # build the api url + def api_url(endpoint) + api_url = @dstk_api_base + endpoint + end - api_url = @api_base + endpoint - arguments_list = arguments.map do |name, value| - name + '=' + URI.encode(value) - end - if arguments_list.length > 0 - arguments_string = '?' + arguments_list.join('&') - api_url += arguments_string - end - response = nil - if !data_payload - response = HTTParty.get(api_url) - else - if data_payload_type == 'json' - data_payload_value = data_payload.to_json - else - data_payload_value = data_payload - end - response = HTTParty.post(api_url, { :body => data_payload_value }) - end + # convert payload to json if type json + def prep_payload(data_payload, data_payload_type) + data_payload_type == 'json' ? data_payload.to_json : data_payload + end + + # item should be an array. if it's not, make it one + def ensure_array(item) + item.is_a?(Array) ? item : [item] + end + ############### end UTILITIES ############### + + def dstk_api_call(endpoint, arguments = {}, data_payload = nil, data_payload_type = 'json') + response = if data_payload + HTTParty.post(api_url(endpoint), + { :body => prep_payload(data_payload, data_payload_type), + :query => arguments }) + else + HTTParty.get(api_url(endpoint), query: arguments) + end - if !response.body or response.code != 200 - raise "DSTK::json_api_call('#{endpoint}', #{arguments.to_json}, '#{data_payload}', '#{data_payload_type}') call to '#{api_url}' failed with code #{response.code} : '#{response.message}'" + unless response.body && response.code == 200 + raise "DSTK::dstk_api_call('#{endpoint}', #{arguments.to_json}, '#{data_payload}', '#{data_payload_type}') call to '#{response.request.uri.to_s}' failed with code #{response.code} : '#{response.message}'" end - json_string = response.body - result = nil begin - result = JSON.parse(json_string) + result = JSON.parse(response.body) + !result.is_a?(Array) && result['error'] ? raise(result['error']) : result rescue JSON::ParseError => e - raise "DSTK::json_api_call('#{endpoint}', #{arguments.to_json}, '#{data_payload}', '#{data_payload_type}') call to '#{api_url}' failed to parse response '#{json_string}' as JSON - #{e.message}" - end - if !result.is_a?(Array) and result['error'] - raise result['error'] + raise "DSTK::dstk_api_call('#{endpoint}', #{arguments.to_json}, '#{data_payload}', '#{data_payload_type}') call to '#{response.request.uri.to_s}' failed to parse response '#{response.body}' as JSON - #{e.message}" end - result end def check_version required_version = 50 - response = json_api_call('/info') + response = dstk_api_call('/info') actual_version = response['version'] if actual_version < required_version raise "DSTK: Version #{actual_version.to_s} found but #{required_version.to_s} is required" end end + def geocode(address) + dstk_api_call('/maps/api/geocode/json', { 'address' => address }) + end + def ip2coordinates(ips) - if !ips.is_a?(Array) then ips = [ips] end - response = json_api_call('/ip2coordinates', {}, ips) - response + dstk_api_call('/ip2coordinates', {}, ensure_array(ips), 'json') end def street2coordinates(addresses) - if !addresses.is_a?(Array) then addresses = [addresses] end - response = json_api_call('/street2coordinates', {}, addresses) - response - end - - def geocode(address) - response = json_api_call('/maps/api/geocode/json', { 'address' => address }) - response + dstk_api_call('/street2coordinates', {}, ensure_array(addresses), 'json') end def coordinates2politics(coordinates) - response = json_api_call('/coordinates2politics', {}, coordinates) - response + dstk_api_call('/coordinates2politics', {}, coordinates, 'json') end def text2places(text) - response = json_api_call('/text2places', {}, text, 'string') - response + dstk_api_call('/text2places', {}, text, 'string') end def file2text(inputfile) - response = json_api_call('/text2places', {}, {:inputfile => inputfile}, 'file') - response + dstk_api_call('/file2text', {}, {:inputfile => inputfile}, 'file') end def text2sentences(text) - response = json_api_call('/text2sentences', {}, text, 'string') - response + dstk_api_call('/text2sentences', {}, text, 'string') end def html2text(html) - response = json_api_call('/html2text', {}, html, 'string') - response + dstk_api_call('/html2text', {}, html, 'string') end def html2story(html) - response = json_api_call('/html2story', {}, html, 'string') - response + dstk_api_call('/html2story', {}, html, 'string') end def text2people(text) - response = json_api_call('/text2people', {}, text, 'string') - response + dstk_api_call('/text2people', {}, text, 'string') end def text2times(text) - response = json_api_call('/text2times', {}, text, 'string') - response + dstk_api_call('/text2times', {}, text, 'string') end def text2sentiment(text) - response = json_api_call('/text2sentiment', {}, text, 'string') - response + dstk_api_call('/text2sentiment', {}, text, 'string') end def coordinates2statistics(coordinates, statistics = nil) @@ -155,8 +135,12 @@ def coordinates2statistics(coordinates, statistics = nil) else arguments = {} end - response = json_api_call('/coordinates2statistics', arguments, coordinates) - response + + dstk_api_call('/coordinates2statistics', arguments, coordinates, 'json') + end + + def twofishes(text) + dstk_api_call('/twofishes', { 'query' => text }) end end diff --git a/rubygem/test/test_dstk.rb b/rubygem/test/test_dstk.rb index 9cda696..d8a73c1 100644 --- a/rubygem/test/test_dstk.rb +++ b/rubygem/test/test_dstk.rb @@ -91,4 +91,11 @@ def test_coordinates2statistics response = @dstk.coordinates2statistics(input, 'population_density') assert_equal expected, response end + + def test_twofishes + input = "pizza القاهرة" + expected = {"interpretations"=> [{"what"=>"pizza", "where"=>"القاهرة", "feature"=> {"cc"=>"EG", "geometry"=> {"center"=>{"lat"=>30.06263, "lng"=>31.24967}, "bounds"=> {"ne"=>{"lat"=>30.1480960846, "lng"=>31.3563537598}, "sw"=>{"lat"=>29.9635601044, "lng"=>31.1625480652}}}, "name"=>"Cairo", "displayName"=>"Cairo, EG", "woeType"=>7, "ids"=>[{"source"=>"geonameid", "id"=>"360630"}], "names"=> [{"name"=>"Cairo", "lang"=>"en", "flags"=>[16, 1]}, {"name"=>"Cairo", "lang"=>"en", "flags"=>[16]}], "highlightedName"=>"القاهرة, EG", "matchedName"=>"القاهرة, EG", "id"=>"geonameid:360630", "attributes"=> {"adm0cap"=>1, "scalerank"=>0, "labelrank"=>3, "natscale"=>600, "population"=>7734614}, "longId"=>"72057594038288566"}, "scores"=>{}}]} + response = @dstk.twofishes(input) + assert_equal expected, response + end end