diff --git a/tests/integration/upload_file_tests/csv/source.csv b/tests/integration/upload_file_tests/csv/source.csv new file mode 100644 index 000000000..a8f87412e --- /dev/null +++ b/tests/integration/upload_file_tests/csv/source.csv @@ -0,0 +1,101 @@ +favorite_color,id,first_name,email,ip_address,updated_at +blue,1,Larry,lking0@miitbeian.gov.cn,'69.135.206.194',2008-09-12 19:08:31 +blue,2,Larry,lperkins1@toplist.cz,'64.210.133.162',1978-05-09 04:15:14 +blue,3,Anna,amontgomery2@miitbeian.gov.cn,'168.104.64.114',2011-10-16 04:07:57 +blue,4,Sandra,sgeorge3@livejournal.com,'229.235.252.98',1973-07-19 10:52:43 +blue,5,Fred,fwoods4@google.cn,'78.229.170.124',2012-09-30 16:38:29 +blue,6,Stephen,shanson5@livejournal.com,'182.227.157.105',1995-11-07 21:40:50 +blue,7,William,wmartinez6@upenn.edu,'135.139.249.50',1982-09-05 03:11:59 +blue,8,Jessica,jlong7@hao123.com,'203.62.178.210',1991-10-16 11:03:15 +blue,9,Douglas,dwhite8@tamu.edu,'178.187.247.1',1979-10-01 09:49:48 +blue,10,Lisa,lcoleman9@nydailynews.com,'168.234.128.249',2011-05-26 07:45:49 +blue,11,Ralph,rfieldsa@home.pl,'55.152.163.149',1972-11-18 19:06:11 +blue,12,Louise,lnicholsb@samsung.com,'141.116.153.154',2014-11-25 20:56:14 +blue,13,Clarence,cduncanc@sfgate.com,'81.171.31.133',2011-11-17 07:02:36 +blue,14,Daniel,dfranklind@omniture.com,'8.204.211.37',1980-09-13 00:09:04 +blue,15,Katherine,klanee@auda.org.au,'176.96.134.59',1997-08-22 19:36:56 +blue,16,Billy,bwardf@wikia.com,'214.108.78.85',2003-10-19 02:14:47 +blue,17,Annie,agarzag@ocn.ne.jp,'190.108.42.70',1988-10-28 15:12:35 +blue,18,Shirley,scolemanh@fastcompany.com,'109.251.164.84',1988-08-24 10:50:57 +blue,19,Roger,rfrazieri@scribd.com,'38.145.218.108',1985-12-31 15:17:15 +blue,20,Lillian,lstanleyj@goodreads.com,'47.57.236.17',1970-06-08 02:09:05 +blue,21,Aaron,arodriguezk@nps.gov,'205.245.118.221',1985-10-11 23:07:49 +blue,22,Patrick,pparkerl@techcrunch.com,'19.8.100.182',2006-03-29 12:53:56 +blue,23,Phillip,pmorenom@intel.com,'41.38.254.103',2011-11-07 15:35:43 +blue,24,Henry,hgarcian@newsvine.com,'1.191.216.252',2008-08-28 08:30:44 +blue,25,Irene,iturnero@opera.com,'50.17.60.190',1994-04-01 07:15:02 +blue,26,Andrew,adunnp@pen.io,'123.52.253.176',2000-11-01 06:03:25 +blue,27,David,dgutierrezq@wp.com,'238.23.203.42',1988-01-25 07:29:18 +blue,28,Henry,hsanchezr@cyberchimps.com,'248.102.2.185',1983-01-01 13:36:37 +blue,29,Evelyn,epetersons@gizmodo.com,'32.80.46.119',1979-07-16 17:24:12 +blue,30,Tammy,tmitchellt@purevolume.com,'249.246.167.88',2001-04-03 10:00:23 +blue,31,Jacqueline,jlittleu@domainmarket.com,'127.181.97.47',1986-02-11 21:35:50 +blue,32,Earl,eortizv@opera.com,'166.47.248.240',1996-07-06 08:16:27 +blue,33,Juan,jgordonw@sciencedirect.com,'71.77.2.200',1987-01-31 03:46:44 +blue,34,Diane,dhowellx@nyu.edu,'140.94.133.12',1994-06-11 02:30:05 +blue,35,Randy,rkennedyy@microsoft.com,'73.255.34.196',2005-05-26 20:28:39 +blue,36,Janice,jriveraz@time.com,'22.214.227.32',1990-02-09 04:16:52 +blue,37,Laura,lperry10@diigo.com,'159.148.145.73',2015-03-17 05:59:25 +blue,38,Gary,gray11@statcounter.com,'40.193.124.56',1970-01-27 10:04:51 +blue,39,Jesse,jmcdonald12@typepad.com,'31.7.86.103',2009-03-14 08:14:29 +blue,40,Sandra,sgonzalez13@goodreads.com,'223.80.168.239',1993-05-21 14:08:54 +blue,41,Scott,smoore14@archive.org,'38.238.46.83',1980-08-30 11:16:56 +blue,42,Phillip,pevans15@cisco.com,'158.234.59.34',2011-12-15 23:26:31 +blue,43,Steven,sriley16@google.ca,'90.247.57.68',2011-10-29 19:03:28 +blue,44,Deborah,dbrown17@hexun.com,'179.125.143.240',1995-04-10 14:36:07 +blue,45,Lori,lross18@ow.ly,'64.80.162.180',1980-12-27 16:49:15 +blue,46,Sean,sjackson19@tumblr.com,'240.116.183.69',1988-06-12 21:24:45 +blue,47,Terry,tbarnes1a@163.com,'118.38.213.137',1997-09-22 16:43:19 +blue,48,Dorothy,dross1b@ebay.com,'116.81.76.49',2005-02-28 13:33:24 +blue,49,Samuel,swashington1c@house.gov,'38.191.253.40',1989-01-19 21:15:48 +blue,50,Ralph,rcarter1d@tinyurl.com,'104.84.60.174',2007-08-11 10:21:49 +green,51,Wayne,whudson1e@princeton.edu,'90.61.24.102',1983-07-03 16:58:12 +green,52,Rose,rjames1f@plala.or.jp,'240.83.81.10',1995-06-08 11:46:23 +green,53,Louise,lcox1g@theglobeandmail.com,'105.11.82.145',2016-09-19 14:45:51 +green,54,Kenneth,kjohnson1h@independent.co.uk,'139.5.45.94',1976-08-17 11:26:19 +green,55,Donna,dbrown1i@amazon.co.uk,'19.45.169.45',2006-05-27 16:51:40 +green,56,Johnny,jvasquez1j@trellian.com,'118.202.238.23',1975-11-17 08:42:32 +green,57,Patrick,pramirez1k@tamu.edu,'231.25.153.198',1997-08-06 11:51:09 +green,58,Helen,hlarson1l@prweb.com,'8.40.21.39',1993-08-04 19:53:40 +green,59,Patricia,pspencer1m@gmpg.org,'212.198.40.15',1977-08-03 16:37:27 +green,60,Joseph,jspencer1n@marriott.com,'13.15.63.238',2005-07-23 20:22:06 +green,61,Phillip,pschmidt1o@blogtalkradio.com,'177.98.201.190',1976-05-19 21:47:44 +green,62,Joan,jwebb1p@google.ru,'105.229.170.71',1972-09-07 17:53:47 +green,63,Phyllis,pkennedy1q@imgur.com,'35.145.8.244',2000-01-01 22:33:37 +green,64,Katherine,khunter1r@smh.com.au,'248.168.205.32',1991-01-09 06:40:24 +green,65,Laura,lvasquez1s@wiley.com,'128.129.115.152',1997-10-23 12:04:56 +green,66,Juan,jdunn1t@state.gov,'44.228.124.51',2004-11-10 05:07:35 +green,67,Judith,jholmes1u@wiley.com,'40.227.179.115',1977-08-02 17:01:45 +green,68,Beverly,bbaker1v@wufoo.com,'208.34.84.59',2016-03-06 20:07:23 +green,69,Lawrence,lcarr1w@flickr.com,'59.158.212.223',1988-09-13 06:07:21 +green,70,Gloria,gwilliams1x@mtv.com,'245.231.88.33',1995-03-18 22:32:46 +green,71,Steven,ssims1y@cbslocal.com,'104.50.58.255',2001-08-05 21:26:20 +green,72,Betty,bmills1z@arstechnica.com,'103.177.214.220',1981-12-14 21:26:54 +green,73,Mildred,mfuller20@prnewswire.com,'151.158.8.130',2000-04-19 10:13:55 +green,74,Donald,dday21@icq.com,'9.178.102.255',1972-12-03 00:58:24 +green,75,Eric,ethomas22@addtoany.com,'85.2.241.227',1992-11-01 05:59:30 +green,76,Joyce,jarmstrong23@sitemeter.com,'169.224.20.36',1985-10-24 06:50:01 +green,77,Maria,mmartinez24@amazonaws.com,'143.189.167.135',2005-10-05 05:17:42 +green,78,Harry,hburton25@youtube.com,'156.47.176.237',1978-03-26 05:53:33 +green,79,Kevin,klawrence26@hao123.com,'79.136.183.83',1994-10-12 04:38:52 +green,80,David,dhall27@prweb.com,'133.149.172.153',1976-12-15 16:24:24 +green,81,Kathy,kperry28@twitter.com,'229.242.72.228',1979-03-04 02:58:56 +green,82,Adam,aprice29@elegantthemes.com,'13.145.21.10',1982-11-07 11:46:59 +green,83,Brandon,bgriffin2a@va.gov,'73.249.128.212',2013-10-30 05:30:36 +green,84,Henry,hnguyen2b@discovery.com,'211.36.214.242',1985-01-09 06:37:27 +green,85,Eric,esanchez2c@edublogs.org,'191.166.188.251',2004-05-01 23:21:42 +green,86,Jason,jlee2d@jimdo.com,'193.92.16.182',1973-01-08 09:05:39 +green,87,Diana,drichards2e@istockphoto.com,'19.130.175.245',1994-10-05 22:50:49 +green,88,Andrea,awelch2f@abc.net.au,'94.155.233.96',2002-04-26 08:41:44 +green,89,Louis,lwagner2g@miitbeian.gov.cn,'26.217.34.111',2003-08-25 07:56:39 +green,90,Jane,jsims2h@seesaa.net,'43.4.220.135',1987-03-20 20:39:04 +green,91,Larry,lgrant2i@si.edu,'97.126.79.34',2000-09-07 20:26:19 +green,92,Louis,ldean2j@prnewswire.com,'37.148.40.127',2011-09-16 20:12:14 +green,93,Jennifer,jcampbell2k@xing.com,'38.106.254.142',1988-07-15 05:06:49 +green,94,Wayne,wcunningham2l@google.com.hk,'223.28.26.187',2009-12-15 06:16:54 +green,95,Lori,lstevens2m@icq.com,'181.250.181.58',1984-10-28 03:29:19 +green,96,Judy,jsimpson2n@marriott.com,'180.121.239.219',1986-02-07 15:18:10 +green,97,Phillip,phoward2o@usa.gov,'255.247.0.175',2002-12-26 08:44:45 +green,98,Gloria,gwalker2p@usa.gov,'156.140.7.128',1997-10-04 07:58:58 +green,99,Paul,pjohnson2q@umn.edu,'183.59.198.197',1991-11-14 12:33:55 +green,100,Frank,fgreene2r@blogspot.com,'150.143.68.121',2010-06-12 23:55:39 diff --git a/tests/integration/upload_file_tests/macros/upload_file.sql b/tests/integration/upload_file_tests/macros/upload_file.sql new file mode 100644 index 000000000..5436170f3 --- /dev/null +++ b/tests/integration/upload_file_tests/macros/upload_file.sql @@ -0,0 +1,3 @@ +{% macro upload_file(local_file_path, database, table_schema, table_name) %} + {% do adapter.upload_file(local_file_path, database, table_schema, table_name, kwargs=kwargs) %} +{% endmacro %} \ No newline at end of file diff --git a/tests/integration/upload_file_tests/ndjson/source.ndjson b/tests/integration/upload_file_tests/ndjson/source.ndjson new file mode 100644 index 000000000..365d8330b --- /dev/null +++ b/tests/integration/upload_file_tests/ndjson/source.ndjson @@ -0,0 +1,100 @@ +{"favorite_color":"blue","id":1,"first_name":"Larry","email":"lking0@miitbeian.gov.cn","ip_address":"'69.135.206.194'","updated_at":"2008-09-12 19:08:31"} +{"favorite_color":"blue","id":2,"first_name":"Larry","email":"lperkins1@toplist.cz","ip_address":"'64.210.133.162'","updated_at":"1978-05-09 04:15:14"} +{"favorite_color":"blue","id":3,"first_name":"Anna","email":"amontgomery2@miitbeian.gov.cn","ip_address":"'168.104.64.114'","updated_at":"2011-10-16 04:07:57"} +{"favorite_color":"blue","id":4,"first_name":"Sandra","email":"sgeorge3@livejournal.com","ip_address":"'229.235.252.98'","updated_at":"1973-07-19 10:52:43"} +{"favorite_color":"blue","id":5,"first_name":"Fred","email":"fwoods4@google.cn","ip_address":"'78.229.170.124'","updated_at":"2012-09-30 16:38:29"} +{"favorite_color":"blue","id":6,"first_name":"Stephen","email":"shanson5@livejournal.com","ip_address":"'182.227.157.105'","updated_at":"1995-11-07 21:40:50"} +{"favorite_color":"blue","id":7,"first_name":"William","email":"wmartinez6@upenn.edu","ip_address":"'135.139.249.50'","updated_at":"1982-09-05 03:11:59"} +{"favorite_color":"blue","id":8,"first_name":"Jessica","email":"jlong7@hao123.com","ip_address":"'203.62.178.210'","updated_at":"1991-10-16 11:03:15"} +{"favorite_color":"blue","id":9,"first_name":"Douglas","email":"dwhite8@tamu.edu","ip_address":"'178.187.247.1'","updated_at":"1979-10-01 09:49:48"} +{"favorite_color":"blue","id":10,"first_name":"Lisa","email":"lcoleman9@nydailynews.com","ip_address":"'168.234.128.249'","updated_at":"2011-05-26 07:45:49"} +{"favorite_color":"blue","id":11,"first_name":"Ralph","email":"rfieldsa@home.pl","ip_address":"'55.152.163.149'","updated_at":"1972-11-18 19:06:11"} +{"favorite_color":"blue","id":12,"first_name":"Louise","email":"lnicholsb@samsung.com","ip_address":"'141.116.153.154'","updated_at":"2014-11-25 20:56:14"} +{"favorite_color":"blue","id":13,"first_name":"Clarence","email":"cduncanc@sfgate.com","ip_address":"'81.171.31.133'","updated_at":"2011-11-17 07:02:36"} +{"favorite_color":"blue","id":14,"first_name":"Daniel","email":"dfranklind@omniture.com","ip_address":"'8.204.211.37'","updated_at":"1980-09-13 00:09:04"} +{"favorite_color":"blue","id":15,"first_name":"Katherine","email":"klanee@auda.org.au","ip_address":"'176.96.134.59'","updated_at":"1997-08-22 19:36:56"} +{"favorite_color":"blue","id":16,"first_name":"Billy","email":"bwardf@wikia.com","ip_address":"'214.108.78.85'","updated_at":"2003-10-19 02:14:47"} +{"favorite_color":"blue","id":17,"first_name":"Annie","email":"agarzag@ocn.ne.jp","ip_address":"'190.108.42.70'","updated_at":"1988-10-28 15:12:35"} +{"favorite_color":"blue","id":18,"first_name":"Shirley","email":"scolemanh@fastcompany.com","ip_address":"'109.251.164.84'","updated_at":"1988-08-24 10:50:57"} +{"favorite_color":"blue","id":19,"first_name":"Roger","email":"rfrazieri@scribd.com","ip_address":"'38.145.218.108'","updated_at":"1985-12-31 15:17:15"} +{"favorite_color":"blue","id":20,"first_name":"Lillian","email":"lstanleyj@goodreads.com","ip_address":"'47.57.236.17'","updated_at":"1970-06-08 02:09:05"} +{"favorite_color":"blue","id":21,"first_name":"Aaron","email":"arodriguezk@nps.gov","ip_address":"'205.245.118.221'","updated_at":"1985-10-11 23:07:49"} +{"favorite_color":"blue","id":22,"first_name":"Patrick","email":"pparkerl@techcrunch.com","ip_address":"'19.8.100.182'","updated_at":"2006-03-29 12:53:56"} +{"favorite_color":"blue","id":23,"first_name":"Phillip","email":"pmorenom@intel.com","ip_address":"'41.38.254.103'","updated_at":"2011-11-07 15:35:43"} +{"favorite_color":"blue","id":24,"first_name":"Henry","email":"hgarcian@newsvine.com","ip_address":"'1.191.216.252'","updated_at":"2008-08-28 08:30:44"} +{"favorite_color":"blue","id":25,"first_name":"Irene","email":"iturnero@opera.com","ip_address":"'50.17.60.190'","updated_at":"1994-04-01 07:15:02"} +{"favorite_color":"blue","id":26,"first_name":"Andrew","email":"adunnp@pen.io","ip_address":"'123.52.253.176'","updated_at":"2000-11-01 06:03:25"} +{"favorite_color":"blue","id":27,"first_name":"David","email":"dgutierrezq@wp.com","ip_address":"'238.23.203.42'","updated_at":"1988-01-25 07:29:18"} +{"favorite_color":"blue","id":28,"first_name":"Henry","email":"hsanchezr@cyberchimps.com","ip_address":"'248.102.2.185'","updated_at":"1983-01-01 13:36:37"} +{"favorite_color":"blue","id":29,"first_name":"Evelyn","email":"epetersons@gizmodo.com","ip_address":"'32.80.46.119'","updated_at":"1979-07-16 17:24:12"} +{"favorite_color":"blue","id":30,"first_name":"Tammy","email":"tmitchellt@purevolume.com","ip_address":"'249.246.167.88'","updated_at":"2001-04-03 10:00:23"} +{"favorite_color":"blue","id":31,"first_name":"Jacqueline","email":"jlittleu@domainmarket.com","ip_address":"'127.181.97.47'","updated_at":"1986-02-11 21:35:50"} +{"favorite_color":"blue","id":32,"first_name":"Earl","email":"eortizv@opera.com","ip_address":"'166.47.248.240'","updated_at":"1996-07-06 08:16:27"} +{"favorite_color":"blue","id":33,"first_name":"Juan","email":"jgordonw@sciencedirect.com","ip_address":"'71.77.2.200'","updated_at":"1987-01-31 03:46:44"} +{"favorite_color":"blue","id":34,"first_name":"Diane","email":"dhowellx@nyu.edu","ip_address":"'140.94.133.12'","updated_at":"1994-06-11 02:30:05"} +{"favorite_color":"blue","id":35,"first_name":"Randy","email":"rkennedyy@microsoft.com","ip_address":"'73.255.34.196'","updated_at":"2005-05-26 20:28:39"} +{"favorite_color":"blue","id":36,"first_name":"Janice","email":"jriveraz@time.com","ip_address":"'22.214.227.32'","updated_at":"1990-02-09 04:16:52"} +{"favorite_color":"blue","id":37,"first_name":"Laura","email":"lperry10@diigo.com","ip_address":"'159.148.145.73'","updated_at":"2015-03-17 05:59:25"} +{"favorite_color":"blue","id":38,"first_name":"Gary","email":"gray11@statcounter.com","ip_address":"'40.193.124.56'","updated_at":"1970-01-27 10:04:51"} +{"favorite_color":"blue","id":39,"first_name":"Jesse","email":"jmcdonald12@typepad.com","ip_address":"'31.7.86.103'","updated_at":"2009-03-14 08:14:29"} +{"favorite_color":"blue","id":40,"first_name":"Sandra","email":"sgonzalez13@goodreads.com","ip_address":"'223.80.168.239'","updated_at":"1993-05-21 14:08:54"} +{"favorite_color":"blue","id":41,"first_name":"Scott","email":"smoore14@archive.org","ip_address":"'38.238.46.83'","updated_at":"1980-08-30 11:16:56"} +{"favorite_color":"blue","id":42,"first_name":"Phillip","email":"pevans15@cisco.com","ip_address":"'158.234.59.34'","updated_at":"2011-12-15 23:26:31"} +{"favorite_color":"blue","id":43,"first_name":"Steven","email":"sriley16@google.ca","ip_address":"'90.247.57.68'","updated_at":"2011-10-29 19:03:28"} +{"favorite_color":"blue","id":44,"first_name":"Deborah","email":"dbrown17@hexun.com","ip_address":"'179.125.143.240'","updated_at":"1995-04-10 14:36:07"} +{"favorite_color":"blue","id":45,"first_name":"Lori","email":"lross18@ow.ly","ip_address":"'64.80.162.180'","updated_at":"1980-12-27 16:49:15"} +{"favorite_color":"blue","id":46,"first_name":"Sean","email":"sjackson19@tumblr.com","ip_address":"'240.116.183.69'","updated_at":"1988-06-12 21:24:45"} +{"favorite_color":"blue","id":47,"first_name":"Terry","email":"tbarnes1a@163.com","ip_address":"'118.38.213.137'","updated_at":"1997-09-22 16:43:19"} +{"favorite_color":"blue","id":48,"first_name":"Dorothy","email":"dross1b@ebay.com","ip_address":"'116.81.76.49'","updated_at":"2005-02-28 13:33:24"} +{"favorite_color":"blue","id":49,"first_name":"Samuel","email":"swashington1c@house.gov","ip_address":"'38.191.253.40'","updated_at":"1989-01-19 21:15:48"} +{"favorite_color":"blue","id":50,"first_name":"Ralph","email":"rcarter1d@tinyurl.com","ip_address":"'104.84.60.174'","updated_at":"2007-08-11 10:21:49"} +{"favorite_color":"green","id":51,"first_name":"Wayne","email":"whudson1e@princeton.edu","ip_address":"'90.61.24.102'","updated_at":"1983-07-03 16:58:12"} +{"favorite_color":"green","id":52,"first_name":"Rose","email":"rjames1f@plala.or.jp","ip_address":"'240.83.81.10'","updated_at":"1995-06-08 11:46:23"} +{"favorite_color":"green","id":53,"first_name":"Louise","email":"lcox1g@theglobeandmail.com","ip_address":"'105.11.82.145'","updated_at":"2016-09-19 14:45:51"} +{"favorite_color":"green","id":54,"first_name":"Kenneth","email":"kjohnson1h@independent.co.uk","ip_address":"'139.5.45.94'","updated_at":"1976-08-17 11:26:19"} +{"favorite_color":"green","id":55,"first_name":"Donna","email":"dbrown1i@amazon.co.uk","ip_address":"'19.45.169.45'","updated_at":"2006-05-27 16:51:40"} +{"favorite_color":"green","id":56,"first_name":"Johnny","email":"jvasquez1j@trellian.com","ip_address":"'118.202.238.23'","updated_at":"1975-11-17 08:42:32"} +{"favorite_color":"green","id":57,"first_name":"Patrick","email":"pramirez1k@tamu.edu","ip_address":"'231.25.153.198'","updated_at":"1997-08-06 11:51:09"} +{"favorite_color":"green","id":58,"first_name":"Helen","email":"hlarson1l@prweb.com","ip_address":"'8.40.21.39'","updated_at":"1993-08-04 19:53:40"} +{"favorite_color":"green","id":59,"first_name":"Patricia","email":"pspencer1m@gmpg.org","ip_address":"'212.198.40.15'","updated_at":"1977-08-03 16:37:27"} +{"favorite_color":"green","id":60,"first_name":"Joseph","email":"jspencer1n@marriott.com","ip_address":"'13.15.63.238'","updated_at":"2005-07-23 20:22:06"} +{"favorite_color":"green","id":61,"first_name":"Phillip","email":"pschmidt1o@blogtalkradio.com","ip_address":"'177.98.201.190'","updated_at":"1976-05-19 21:47:44"} +{"favorite_color":"green","id":62,"first_name":"Joan","email":"jwebb1p@google.ru","ip_address":"'105.229.170.71'","updated_at":"1972-09-07 17:53:47"} +{"favorite_color":"green","id":63,"first_name":"Phyllis","email":"pkennedy1q@imgur.com","ip_address":"'35.145.8.244'","updated_at":"2000-01-01 22:33:37"} +{"favorite_color":"green","id":64,"first_name":"Katherine","email":"khunter1r@smh.com.au","ip_address":"'248.168.205.32'","updated_at":"1991-01-09 06:40:24"} +{"favorite_color":"green","id":65,"first_name":"Laura","email":"lvasquez1s@wiley.com","ip_address":"'128.129.115.152'","updated_at":"1997-10-23 12:04:56"} +{"favorite_color":"green","id":66,"first_name":"Juan","email":"jdunn1t@state.gov","ip_address":"'44.228.124.51'","updated_at":"2004-11-10 05:07:35"} +{"favorite_color":"green","id":67,"first_name":"Judith","email":"jholmes1u@wiley.com","ip_address":"'40.227.179.115'","updated_at":"1977-08-02 17:01:45"} +{"favorite_color":"green","id":68,"first_name":"Beverly","email":"bbaker1v@wufoo.com","ip_address":"'208.34.84.59'","updated_at":"2016-03-06 20:07:23"} +{"favorite_color":"green","id":69,"first_name":"Lawrence","email":"lcarr1w@flickr.com","ip_address":"'59.158.212.223'","updated_at":"1988-09-13 06:07:21"} +{"favorite_color":"green","id":70,"first_name":"Gloria","email":"gwilliams1x@mtv.com","ip_address":"'245.231.88.33'","updated_at":"1995-03-18 22:32:46"} +{"favorite_color":"green","id":71,"first_name":"Steven","email":"ssims1y@cbslocal.com","ip_address":"'104.50.58.255'","updated_at":"2001-08-05 21:26:20"} +{"favorite_color":"green","id":72,"first_name":"Betty","email":"bmills1z@arstechnica.com","ip_address":"'103.177.214.220'","updated_at":"1981-12-14 21:26:54"} +{"favorite_color":"green","id":73,"first_name":"Mildred","email":"mfuller20@prnewswire.com","ip_address":"'151.158.8.130'","updated_at":"2000-04-19 10:13:55"} +{"favorite_color":"green","id":74,"first_name":"Donald","email":"dday21@icq.com","ip_address":"'9.178.102.255'","updated_at":"1972-12-03 00:58:24"} +{"favorite_color":"green","id":75,"first_name":"Eric","email":"ethomas22@addtoany.com","ip_address":"'85.2.241.227'","updated_at":"1992-11-01 05:59:30"} +{"favorite_color":"green","id":76,"first_name":"Joyce","email":"jarmstrong23@sitemeter.com","ip_address":"'169.224.20.36'","updated_at":"1985-10-24 06:50:01"} +{"favorite_color":"green","id":77,"first_name":"Maria","email":"mmartinez24@amazonaws.com","ip_address":"'143.189.167.135'","updated_at":"2005-10-05 05:17:42"} +{"favorite_color":"green","id":78,"first_name":"Harry","email":"hburton25@youtube.com","ip_address":"'156.47.176.237'","updated_at":"1978-03-26 05:53:33"} +{"favorite_color":"green","id":79,"first_name":"Kevin","email":"klawrence26@hao123.com","ip_address":"'79.136.183.83'","updated_at":"1994-10-12 04:38:52"} +{"favorite_color":"green","id":80,"first_name":"David","email":"dhall27@prweb.com","ip_address":"'133.149.172.153'","updated_at":"1976-12-15 16:24:24"} +{"favorite_color":"green","id":81,"first_name":"Kathy","email":"kperry28@twitter.com","ip_address":"'229.242.72.228'","updated_at":"1979-03-04 02:58:56"} +{"favorite_color":"green","id":82,"first_name":"Adam","email":"aprice29@elegantthemes.com","ip_address":"'13.145.21.10'","updated_at":"1982-11-07 11:46:59"} +{"favorite_color":"green","id":83,"first_name":"Brandon","email":"bgriffin2a@va.gov","ip_address":"'73.249.128.212'","updated_at":"2013-10-30 05:30:36"} +{"favorite_color":"green","id":84,"first_name":"Henry","email":"hnguyen2b@discovery.com","ip_address":"'211.36.214.242'","updated_at":"1985-01-09 06:37:27"} +{"favorite_color":"green","id":85,"first_name":"Eric","email":"esanchez2c@edublogs.org","ip_address":"'191.166.188.251'","updated_at":"2004-05-01 23:21:42"} +{"favorite_color":"green","id":86,"first_name":"Jason","email":"jlee2d@jimdo.com","ip_address":"'193.92.16.182'","updated_at":"1973-01-08 09:05:39"} +{"favorite_color":"green","id":87,"first_name":"Diana","email":"drichards2e@istockphoto.com","ip_address":"'19.130.175.245'","updated_at":"1994-10-05 22:50:49"} +{"favorite_color":"green","id":88,"first_name":"Andrea","email":"awelch2f@abc.net.au","ip_address":"'94.155.233.96'","updated_at":"2002-04-26 08:41:44"} +{"favorite_color":"green","id":89,"first_name":"Louis","email":"lwagner2g@miitbeian.gov.cn","ip_address":"'26.217.34.111'","updated_at":"2003-08-25 07:56:39"} +{"favorite_color":"green","id":90,"first_name":"Jane","email":"jsims2h@seesaa.net","ip_address":"'43.4.220.135'","updated_at":"1987-03-20 20:39:04"} +{"favorite_color":"green","id":91,"first_name":"Larry","email":"lgrant2i@si.edu","ip_address":"'97.126.79.34'","updated_at":"2000-09-07 20:26:19"} +{"favorite_color":"green","id":92,"first_name":"Louis","email":"ldean2j@prnewswire.com","ip_address":"'37.148.40.127'","updated_at":"2011-09-16 20:12:14"} +{"favorite_color":"green","id":93,"first_name":"Jennifer","email":"jcampbell2k@xing.com","ip_address":"'38.106.254.142'","updated_at":"1988-07-15 05:06:49"} +{"favorite_color":"green","id":94,"first_name":"Wayne","email":"wcunningham2l@google.com.hk","ip_address":"'223.28.26.187'","updated_at":"2009-12-15 06:16:54"} +{"favorite_color":"green","id":95,"first_name":"Lori","email":"lstevens2m@icq.com","ip_address":"'181.250.181.58'","updated_at":"1984-10-28 03:29:19"} +{"favorite_color":"green","id":96,"first_name":"Judy","email":"jsimpson2n@marriott.com","ip_address":"'180.121.239.219'","updated_at":"1986-02-07 15:18:10"} +{"favorite_color":"green","id":97,"first_name":"Phillip","email":"phoward2o@usa.gov","ip_address":"'255.247.0.175'","updated_at":"2002-12-26 08:44:45"} +{"favorite_color":"green","id":98,"first_name":"Gloria","email":"gwalker2p@usa.gov","ip_address":"'156.140.7.128'","updated_at":"1997-10-04 07:58:58"} +{"favorite_color":"green","id":99,"first_name":"Paul","email":"pjohnson2q@umn.edu","ip_address":"'183.59.198.197'","updated_at":"1991-11-14 12:33:55"} +{"favorite_color":"green","id":100,"first_name":"Frank","email":"fgreene2r@blogspot.com","ip_address":"'150.143.68.121'","updated_at":"2010-06-12 23:55:39"} \ No newline at end of file diff --git a/tests/integration/upload_file_tests/parquet/source.parquet b/tests/integration/upload_file_tests/parquet/source.parquet new file mode 100644 index 000000000..408ad6207 Binary files /dev/null and b/tests/integration/upload_file_tests/parquet/source.parquet differ diff --git a/tests/integration/upload_file_tests/test_upload_file.py b/tests/integration/upload_file_tests/test_upload_file.py new file mode 100644 index 000000000..3a874a13a --- /dev/null +++ b/tests/integration/upload_file_tests/test_upload_file.py @@ -0,0 +1,73 @@ +from tests.integration.base import DBTIntegrationTest, use_profile +import yaml + + +class TestUploadFileCSV(DBTIntegrationTest): + @property + def schema(self): + return "adapter_methods_caching" + + @property + def models(self): + return "models" + + @use_profile('bigquery') + def test_bigquery_upload_file_csv(self): + upload_args = yaml.safe_dump({ + 'local_file_path': './csv/source.csv', + 'database': self.default_database, + 'table_schema': self.unique_schema(), + 'table_name': 'TestUploadFileCSV', + 'skip_leading_rows': 1, + 'autodetect': True, + 'write_disposition': 'WRITE_TRUNCATE' + }) + result = self.run_dbt(['run-operation', 'upload_file', '--args', upload_args]) + self.assertTrue(result.success) + + +class TestUploadFileNDJSON(DBTIntegrationTest): + @property + def schema(self): + return "adapter_methods_caching" + + @property + def models(self): + return "models" + + @use_profile('bigquery') + def test_bigquery_upload_file_ndjson(self): + upload_args = yaml.safe_dump({ + 'local_file_path': './ndjson/source.ndjson', + 'database': self.default_database, + 'table_schema': self.unique_schema(), + 'table_name': 'TestUploadFileNDJSON', + 'autodetect': True, + 'source_format': 'NEWLINE_DELIMITED_JSON', + 'write_disposition': 'WRITE_TRUNCATE' + }) + result = self.run_dbt(['run-operation', 'upload_file', '--args', upload_args]) + self.assertTrue(result.success) + + +class TestUploadFileParquet(DBTIntegrationTest): + @property + def schema(self): + return "adapter_methods_caching" + + @property + def models(self): + return "models" + + @use_profile('bigquery') + def test_bigquery_upload_file_parquet(self): + upload_args = yaml.safe_dump({ + 'local_file_path': './parquet/source.parquet', + 'database': self.default_database, + 'table_schema': self.unique_schema(), + 'table_name': 'TestUploadFileParquet', + 'source_format': 'PARQUET', + 'write_disposition': 'WRITE_TRUNCATE' + }) + result = self.run_dbt(['run-operation', 'upload_file', '--args', upload_args]) + self.assertTrue(result.success) \ No newline at end of file