google.php 71 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612
  1. <?php
  2. // todo:
  3. // aliexpress tracking links
  4. // enhanced msx notice
  5. // detect "sorry" page
  6. class google{
  7. private const is_class = ".";
  8. private const is_id = "#";
  9. public function __construct(){
  10. include "lib/fuckhtml.php";
  11. $this->fuckhtml = new fuckhtml();
  12. include "lib/backend.php";
  13. $this->backend = new backend("google");
  14. }
  15. public function getfilters($page){
  16. $base = [
  17. "country" => [ // gl=<country>
  18. "display" => "Country",
  19. "option" => [
  20. "any" => "Instance's country",
  21. "af" => "Afghanistan",
  22. "al" => "Albania",
  23. "dz" => "Algeria",
  24. "as" => "American Samoa",
  25. "ad" => "Andorra",
  26. "ao" => "Angola",
  27. "ai" => "Anguilla",
  28. "aq" => "Antarctica",
  29. "ag" => "Antigua and Barbuda",
  30. "ar" => "Argentina",
  31. "am" => "Armenia",
  32. "aw" => "Aruba",
  33. "au" => "Australia",
  34. "at" => "Austria",
  35. "az" => "Azerbaijan",
  36. "bs" => "Bahamas",
  37. "bh" => "Bahrain",
  38. "bd" => "Bangladesh",
  39. "bb" => "Barbados",
  40. "by" => "Belarus",
  41. "be" => "Belgium",
  42. "bz" => "Belize",
  43. "bj" => "Benin",
  44. "bm" => "Bermuda",
  45. "bt" => "Bhutan",
  46. "bo" => "Bolivia",
  47. "ba" => "Bosnia and Herzegovina",
  48. "bw" => "Botswana",
  49. "bv" => "Bouvet Island",
  50. "br" => "Brazil",
  51. "io" => "British Indian Ocean Territory",
  52. "bn" => "Brunei Darussalam",
  53. "bg" => "Bulgaria",
  54. "bf" => "Burkina Faso",
  55. "bi" => "Burundi",
  56. "kh" => "Cambodia",
  57. "cm" => "Cameroon",
  58. "ca" => "Canada",
  59. "cv" => "Cape Verde",
  60. "ky" => "Cayman Islands",
  61. "cf" => "Central African Republic",
  62. "td" => "Chad",
  63. "cl" => "Chile",
  64. "cn" => "China",
  65. "cx" => "Christmas Island",
  66. "cc" => "Cocos (Keeling) Islands",
  67. "co" => "Colombia",
  68. "km" => "Comoros",
  69. "cg" => "Congo",
  70. "cd" => "Congo, the Democratic Republic",
  71. "ck" => "Cook Islands",
  72. "cr" => "Costa Rica",
  73. "ci" => "Cote D'ivoire",
  74. "hr" => "Croatia",
  75. "cu" => "Cuba",
  76. "cy" => "Cyprus",
  77. "cz" => "Czech Republic",
  78. "dk" => "Denmark",
  79. "dj" => "Djibouti",
  80. "dm" => "Dominica",
  81. "do" => "Dominican Republic",
  82. "ec" => "Ecuador",
  83. "eg" => "Egypt",
  84. "sv" => "El Salvador",
  85. "gq" => "Equatorial Guinea",
  86. "er" => "Eritrea",
  87. "ee" => "Estonia",
  88. "et" => "Ethiopia",
  89. "fk" => "Falkland Islands (Malvinas)",
  90. "fo" => "Faroe Islands",
  91. "fj" => "Fiji",
  92. "fi" => "Finland",
  93. "fr" => "France",
  94. "gf" => "French Guiana",
  95. "pf" => "French Polynesia",
  96. "tf" => "French Southern Territories",
  97. "ga" => "Gabon",
  98. "gm" => "Gambia",
  99. "ge" => "Georgia",
  100. "de" => "Germany",
  101. "gh" => "Ghana",
  102. "gi" => "Gibraltar",
  103. "gr" => "Greece",
  104. "gl" => "Greenland",
  105. "gd" => "Grenada",
  106. "gp" => "Guadeloupe",
  107. "gu" => "Guam",
  108. "gt" => "Guatemala",
  109. "gn" => "Guinea",
  110. "gw" => "Guinea-Bissau",
  111. "gy" => "Guyana",
  112. "ht" => "Haiti",
  113. "hm" => "Heard Island and Mcdonald Islands",
  114. "va" => "Holy See (Vatican City State)",
  115. "hn" => "Honduras",
  116. "hk" => "Hong Kong",
  117. "hu" => "Hungary",
  118. "is" => "Iceland",
  119. "in" => "India",
  120. "id" => "Indonesia",
  121. "ir" => "Iran, Islamic Republic",
  122. "iq" => "Iraq",
  123. "ie" => "Ireland",
  124. "il" => "Israel",
  125. "it" => "Italy",
  126. "jm" => "Jamaica",
  127. "jp" => "Japan",
  128. "jo" => "Jordan",
  129. "kz" => "Kazakhstan",
  130. "ke" => "Kenya",
  131. "ki" => "Kiribati",
  132. "kp" => "Korea, Democratic People's Republic",
  133. "kr" => "Korea, Republic",
  134. "kw" => "Kuwait",
  135. "kg" => "Kyrgyzstan",
  136. "la" => "Lao People's Democratic Republic",
  137. "lv" => "Latvia",
  138. "lb" => "Lebanon",
  139. "ls" => "Lesotho",
  140. "lr" => "Liberia",
  141. "ly" => "Libyan Arab Jamahiriya",
  142. "li" => "Liechtenstein",
  143. "lt" => "Lithuania",
  144. "lu" => "Luxembourg",
  145. "mo" => "Macao",
  146. "mk" => "Macedonia, the Former Yugosalv Republic",
  147. "mg" => "Madagascar",
  148. "mw" => "Malawi",
  149. "my" => "Malaysia",
  150. "mv" => "Maldives",
  151. "ml" => "Mali",
  152. "mt" => "Malta",
  153. "mh" => "Marshall Islands",
  154. "mq" => "Martinique",
  155. "mr" => "Mauritania",
  156. "mu" => "Mauritius",
  157. "yt" => "Mayotte",
  158. "mx" => "Mexico",
  159. "fm" => "Micronesia, Federated States",
  160. "md" => "Moldova, Republic",
  161. "mc" => "Monaco",
  162. "mn" => "Mongolia",
  163. "ms" => "Montserrat",
  164. "ma" => "Morocco",
  165. "mz" => "Mozambique",
  166. "mm" => "Myanmar",
  167. "na" => "Namibia",
  168. "nr" => "Nauru",
  169. "np" => "Nepal",
  170. "nl" => "Netherlands",
  171. "an" => "Netherlands Antilles",
  172. "nc" => "New Caledonia",
  173. "nz" => "New Zealand",
  174. "ni" => "Nicaragua",
  175. "ne" => "Niger",
  176. "ng" => "Nigeria",
  177. "nu" => "Niue",
  178. "nf" => "Norfolk Island",
  179. "mp" => "Northern Mariana Islands",
  180. "no" => "Norway",
  181. "om" => "Oman",
  182. "pk" => "Pakistan",
  183. "pw" => "Palau",
  184. "ps" => "Palestinian Territory, Occupied",
  185. "pa" => "Panama",
  186. "pg" => "Papua New Guinea",
  187. "py" => "Paraguay",
  188. "pe" => "Peru",
  189. "ph" => "Philippines",
  190. "pn" => "Pitcairn",
  191. "pl" => "Poland",
  192. "pt" => "Portugal",
  193. "pr" => "Puerto Rico",
  194. "qa" => "Qatar",
  195. "re" => "Reunion",
  196. "ro" => "Romania",
  197. "ru" => "Russian Federation",
  198. "rw" => "Rwanda",
  199. "sh" => "Saint Helena",
  200. "kn" => "Saint Kitts and Nevis",
  201. "lc" => "Saint Lucia",
  202. "pm" => "Saint Pierre and Miquelon",
  203. "vc" => "Saint Vincent and the Grenadines",
  204. "ws" => "Samoa",
  205. "sm" => "San Marino",
  206. "st" => "Sao Tome and Principe",
  207. "sa" => "Saudi Arabia",
  208. "sn" => "Senegal",
  209. "cs" => "Serbia and Montenegro",
  210. "sc" => "Seychelles",
  211. "sl" => "Sierra Leone",
  212. "sg" => "Singapore",
  213. "sk" => "Slovakia",
  214. "si" => "Slovenia",
  215. "sb" => "Solomon Islands",
  216. "so" => "Somalia",
  217. "za" => "South Africa",
  218. "gs" => "South Georgia and the South Sandwich Islands",
  219. "es" => "Spain",
  220. "lk" => "Sri Lanka",
  221. "sd" => "Sudan",
  222. "sr" => "Suriname",
  223. "sj" => "Svalbard and Jan Mayen",
  224. "sz" => "Swaziland",
  225. "se" => "Sweden",
  226. "ch" => "Switzerland",
  227. "sy" => "Syrian Arab Republic",
  228. "tw" => "Taiwan, Province of China",
  229. "tj" => "Tajikistan",
  230. "tz" => "Tanzania, United Republic",
  231. "th" => "Thailand",
  232. "tl" => "Timor-Leste",
  233. "tg" => "Togo",
  234. "tk" => "Tokelau",
  235. "to" => "Tonga",
  236. "tt" => "Trinidad and Tobago",
  237. "tn" => "Tunisia",
  238. "tr" => "Turkey",
  239. "tm" => "Turkmenistan",
  240. "tc" => "Turks and Caicos Islands",
  241. "tv" => "Tuvalu",
  242. "ug" => "Uganda",
  243. "ua" => "Ukraine",
  244. "ae" => "United Arab Emirates",
  245. "uk" => "United Kingdom",
  246. "us" => "United States",
  247. "um" => "United States Minor Outlying Islands",
  248. "uy" => "Uruguay",
  249. "uz" => "Uzbekistan",
  250. "vu" => "Vanuatu",
  251. "ve" => "Venezuela",
  252. "vn" => "Viet Nam",
  253. "vg" => "Virgin Islands, British",
  254. "vi" => "Virgin Islands, U.S.",
  255. "wf" => "Wallis and Futuna",
  256. "eh" => "Western Sahara",
  257. "ye" => "Yemen",
  258. "zm" => "Zambia",
  259. "zw" => "Zimbabwe"
  260. ]
  261. ],
  262. "nsfw" => [
  263. "display" => "NSFW",
  264. "option" => [
  265. "yes" => "Yes", // safe=active
  266. "no" => "No" // safe=off
  267. ]
  268. ],
  269. "lang" => [ // lr=<lang> (prefix lang with "lang_")
  270. "display" => "Language",
  271. "option" => [
  272. "any" => "Any language",
  273. "ar" => "Arabic",
  274. "bg" => "Bulgarian",
  275. "ca" => "Catalan",
  276. "cs" => "Czech",
  277. "da" => "Danish",
  278. "de" => "German",
  279. "el" => "Greek",
  280. "en" => "English",
  281. "es" => "Spanish",
  282. "et" => "Estonian",
  283. "fi" => "Finnish",
  284. "fr" => "French",
  285. "hr" => "Croatian",
  286. "hu" => "Hungarian",
  287. "id" => "Indonesian",
  288. "is" => "Icelandic",
  289. "it" => "Italian",
  290. "iw" => "Hebrew",
  291. "ja" => "Japanese",
  292. "ko" => "Korean",
  293. "lt" => "Lithuanian",
  294. "lv" => "Latvian",
  295. "nl" => "Dutch",
  296. "no" => "Norwegian",
  297. "pl" => "Polish",
  298. "pt" => "Portuguese",
  299. "ro" => "Romanian",
  300. "ru" => "Russian",
  301. "sk" => "Slovak",
  302. "sl" => "Slovenian",
  303. "sr" => "Serbian",
  304. "sv" => "Swedish",
  305. "tr" => "Turkish",
  306. "zh-CN" => "Chinese (Simplified)",
  307. "zh-TW" => "Chinese (Traditional)"
  308. ]
  309. ]
  310. ];
  311. switch($page){
  312. case "web":
  313. return array_merge(
  314. $base,
  315. [
  316. "newer" => [ // &sort=review-date:r:20090301:20090430
  317. "display" => "Newer than",
  318. "option" => "_DATE"
  319. ],
  320. "older" => [
  321. "display" => "Older than",
  322. "option" => "_DATE"
  323. ]
  324. ]
  325. );
  326. break;
  327. case "images":
  328. return array_merge(
  329. $base,
  330. [
  331. "time" => [ // tbs=qrd:<size>
  332. "display" => "Time posted",
  333. "option" => [
  334. "any" => "Any time",
  335. "d" => "Past 24 hours",
  336. "w" => "Past week",
  337. "m" => "Past month",
  338. "y" => "Past year"
  339. ]
  340. ],
  341. "size" => [
  342. "display" => "Size",
  343. "option" => [
  344. // tbs=isz:<size>
  345. "any" => "Any size",
  346. "l" => "Large",
  347. "m" => "Medium",
  348. "i" => "Icon",
  349. // from here
  350. // tbz:lt,islt:<size>
  351. "qsvga" => "Larger than 400x300",
  352. "vga" => "Larger than 640x480",
  353. "qsvga" => "Larger than 800x600",
  354. "xga" => "Larger than 1024x768",
  355. "2mp" => "Larger than 2MP",
  356. "4mp" => "Larger than 4MP",
  357. "6mp" => "Larger than 6MP",
  358. "8mp" => "Larger than 8MP",
  359. "10mp" => "Larger than 10MP",
  360. "12mp" => "Larger than 12MP",
  361. "15mp" => "Larger than 15MP",
  362. "20mp" => "Larger than 20MP",
  363. "40mp" => "Larger than 40MP",
  364. "70mp" => "Larger than 70MP"
  365. ]
  366. ],
  367. "ratio" => [ // tbs=iar:<size>
  368. "display" => "Aspect ratio",
  369. "option" => [
  370. "any" => "Any ratio",
  371. "t" => "Tall",
  372. "s" => "Square",
  373. "w" => "Wide",
  374. "xw" => "Panoramic"
  375. ]
  376. ],
  377. "color" => [ // tbs=ic:<color>
  378. "display" => "Color",
  379. "option" => [
  380. "any" => "Any color",
  381. "color" => "Full color",
  382. "gray" => "Black & white",
  383. "trans" => "Transparent",
  384. // from there, its ic:specific,isc:<color>
  385. "red" => "Red",
  386. "orange" => "Orange",
  387. "yellow" => "Yellow",
  388. "green" => "Green",
  389. "teal" => "Teal",
  390. "blue" => "Blue",
  391. "purple" => "Purple",
  392. "pink" => "Pink",
  393. "white" => "White",
  394. "gray" => "Gray",
  395. "black" => "Black",
  396. "brown" => "Brown"
  397. ]
  398. ],
  399. "type" => [ // tbs=itp:<type>
  400. "display" => "Type",
  401. "option" => [
  402. "any" => "Any type",
  403. "face" => "Faces",
  404. "clipart" => "Clip Art",
  405. "lineart" => "Line Drawing",
  406. "stock" => "Stock",
  407. "animated" => "Animated"
  408. ]
  409. ],
  410. "format" => [ // tbs=ift:<format>
  411. "display" => "Format",
  412. "option" => [
  413. "any" => "Any format",
  414. "jpg" => "JPG",
  415. "gif" => "GIF",
  416. "png" => "PNG",
  417. "bmp" => "BMP",
  418. "svg" => "SVG",
  419. "webp" => "WEBP",
  420. "ico" => "ICO",
  421. "craw" => "RAW"
  422. ]
  423. ],
  424. "rights" => [ // tbs=il:<rights>
  425. "display" => "Usage rights",
  426. "option" => [
  427. "any" => "Any license",
  428. "cl" => "Creative Commons licenses",
  429. "ol" => "Commercial & other licenses"
  430. ]
  431. ]
  432. ]
  433. );
  434. break;
  435. case "videos":
  436. return array_merge(
  437. $base,
  438. [
  439. "time" => [
  440. "display" => "Time posted",
  441. "option" => [ // tbs=qdr
  442. "any" => "Any time",
  443. "h" => "Past hour",
  444. "d" => "Past 24 hours",
  445. "w" => "Past week",
  446. "m" => "Past month",
  447. "y" => "Past year"
  448. ]
  449. ],
  450. "duration" => [
  451. "display" => "Duration",
  452. "option" => [
  453. "any" => "Any duration",
  454. "s" => "Short (0-4min)", // tbs=dur:s
  455. "m" => "Medium (4-20min)", // tbs=dur:m
  456. "l" => "Long (20+ min)" // tbs=dur:l
  457. ]
  458. ],
  459. "quality" => [
  460. "display" => "Quality",
  461. "option" => [
  462. "any" => "Any quality",
  463. "h" => "High quality" // tbs=hq:h
  464. ]
  465. ],
  466. "captions" => [
  467. "display" => "Captions",
  468. "option" => [
  469. "any" => "No preference",
  470. "yes" => "Closed captioned" // tbs=cc:1
  471. ]
  472. ]
  473. ]
  474. );
  475. break;
  476. case "news":
  477. return array_merge(
  478. $base,
  479. [
  480. "time" => [
  481. "display" => "Time posted",
  482. "option" => [ // tbs=qdr
  483. "any" => "Any time",
  484. "h" => "Past hour",
  485. "d" => "Past 24 hours",
  486. "w" => "Past week",
  487. "m" => "Past month",
  488. "y" => "Past year",
  489. "a" => "Archives" // tbs=ar:1
  490. ]
  491. ],
  492. "sort" => [
  493. "display" => "Sort",
  494. "option" => [
  495. "relevance" => "Relevance",
  496. "date" => "Date" // sbd:1
  497. ]
  498. ]
  499. ]
  500. );
  501. break;
  502. }
  503. }
  504. private function get($proxy, $url, $get = []){
  505. $headers = [
  506. "User-Agent: Mozilla/5.0 (Linux; U; Android 2.3.3; pt-pt; LG-P500h-parrot Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1 MMS/LG-Android-MMS-V1.0/1.2",
  507. "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
  508. "Accept-Language: en-US,en;q=0.5",
  509. "Accept-Encoding: gzip",
  510. "DNT: 1",
  511. "Connection: keep-alive",
  512. "Upgrade-Insecure-Requests: 1",
  513. "Sec-Fetch-Dest: document",
  514. "Sec-Fetch-Mode: navigate",
  515. "Sec-Fetch-Site: none",
  516. "Sec-Fetch-User: ?1"
  517. ];
  518. $curlproc = curl_init();
  519. if($get !== []){
  520. $get = http_build_query($get);
  521. $url .= "?" . $get;
  522. }
  523. curl_setopt($curlproc, CURLOPT_URL, $url);
  524. curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
  525. curl_setopt($curlproc, CURLOPT_HTTPHEADER, $headers);
  526. curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
  527. curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
  528. curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
  529. curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
  530. curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
  531. $this->backend->assign_proxy($curlproc, $proxy);
  532. $data = curl_exec($curlproc);
  533. if(curl_errno($curlproc)){
  534. throw new Exception(curl_error($curlproc));
  535. }
  536. curl_close($curlproc);
  537. return $data;
  538. }
  539. public function web($get){
  540. if($get["npt"]){
  541. [$req, $ip] = $this->backend->get($get["npt"], "web");
  542. parse_str(
  543. parse_url($req, PHP_URL_QUERY),
  544. $search
  545. );
  546. if(isset($search["q"])){
  547. $search = $search["q"];
  548. }else{
  549. $search = "a"; // lol
  550. }
  551. try{
  552. $html =
  553. $this->get(
  554. $ip,
  555. "https://www.google.com" . $req,
  556. []
  557. );
  558. }catch(Exception $error){
  559. throw new Exception("Failed to get HTML");
  560. }
  561. }else{
  562. $search = $get["s"];
  563. $country = $get["country"];
  564. $nsfw = $get["nsfw"];
  565. $lang = $get["lang"];
  566. $older = $get["older"];
  567. $newer = $get["newer"];
  568. $ip = $this->backend->get_ip();
  569. $params = [
  570. "q" => $search,
  571. "hl" => "en",
  572. "num" => 20 // get 20 results
  573. ];
  574. // country
  575. if($country != "any"){
  576. $params["gl"] = $country;
  577. }
  578. // nsfw
  579. $params["safe"] = $nsfw == "yes" ? "off" : "active";
  580. // language
  581. if($lang != "any"){
  582. $params["lr"] = "lang_" . $lang;
  583. }
  584. // &sort=review-date:r:20090301:20090430
  585. $older = $older === false ? false : date("Ymd", $older);
  586. $newer = $newer === false ? false : date("Ymd", $newer);
  587. if(
  588. $older !== false &&
  589. $newer === false
  590. ){
  591. $newer = date("Ymd", time());
  592. }
  593. if(
  594. $older !== false ||
  595. $newer !== false
  596. ){
  597. $params["sort"] = "review-date:r:" . $older . ":" . $newer;
  598. }
  599. try{
  600. $html =
  601. $this->get(
  602. $ip,
  603. "https://www.google.com/search",
  604. $params
  605. );
  606. }catch(Exception $error){
  607. throw new Exception("Failed to get HTML");
  608. }
  609. }
  610. return $this->parsepage($html, "web", $search, $ip);
  611. }
  612. public function video($get){
  613. if($get["npt"]){
  614. [$req, $ip] = $this->backend->get($get["npt"], "videos");
  615. parse_str(
  616. parse_url($req, PHP_URL_QUERY),
  617. $search
  618. );
  619. if(isset($search["q"])){
  620. $search = $search["q"];
  621. }else{
  622. $search = "a"; // lol
  623. }
  624. try{
  625. $html =
  626. $this->get(
  627. $ip,
  628. "https://www.google.com" . $req,
  629. []
  630. );
  631. }catch(Exception $error){
  632. throw new Exception("Failed to get HTML");
  633. }
  634. }else{
  635. $search = $get["s"];
  636. $country = $get["country"];
  637. $nsfw = $get["nsfw"];
  638. $lang = $get["lang"];
  639. $time = $get["time"];
  640. $duration = $get["duration"];
  641. $quality = $get["quality"];
  642. $captions = $get["captions"];
  643. $ip = $this->backend->get_ip();
  644. $params = [
  645. "q" => $search,
  646. "tbm" => "vid",
  647. "hl" => "en",
  648. "num" => "20"
  649. ];
  650. // country
  651. if($country != "any"){
  652. $params["gl"] = $country;
  653. }
  654. // nsfw
  655. $params["safe"] = $nsfw == "yes" ? "off" : "active";
  656. // language
  657. if($lang != "any"){
  658. $params["lr"] = "lang_" . $lang;
  659. }
  660. $tbs = [];
  661. // time
  662. if($time != "any"){
  663. $tbs[] = "qdr:" . $time;
  664. }
  665. // duration
  666. if($duration != "any"){
  667. $tbs[] = "dur:" . $duration;
  668. }
  669. // quality
  670. if($quality != "any"){
  671. $tbs[] = "hq:" . $quality;
  672. }
  673. // captions
  674. if($captions != "any"){
  675. $tbs[] = "cc:" . $captions;
  676. }
  677. // append tbs
  678. if(count($tbs) !== 0){
  679. $params["tbs"] =
  680. implode(",", $tbs);
  681. }
  682. try{
  683. $html =
  684. $this->get(
  685. $ip,
  686. "https://www.google.com/search",
  687. $params
  688. );
  689. }catch(Exception $error){
  690. throw new Exception("Failed to get HTML");
  691. }
  692. }
  693. $json = $this->parsepage($html, "videos", $search, $ip);
  694. $out = [
  695. "status" => "ok",
  696. "npt" => $json["npt"],
  697. "video" => [],
  698. "author" => [],
  699. "livestream" => [],
  700. "playlist" => [],
  701. "reel" => []
  702. ];
  703. foreach($json["web"] as $item){
  704. $out["video"][] = [
  705. "title" => $item["title"],
  706. "description" => $item["description"],
  707. "author" => [
  708. "name" => null,
  709. "url" => null,
  710. "avatar" => null
  711. ],
  712. "date" => isset($item["table"]["Posted"]) ? strtotime($item["table"]["Posted"]) : null,
  713. "duration" => isset($item["table"]["Duration"]) ? $this->hms2int($item["table"]["Duration"]) : null,
  714. "views" => null,
  715. "thumb" =>
  716. $item["thumb"]["url"] === null ?
  717. [
  718. "url" => null,
  719. "ratio" => null
  720. ] :
  721. [
  722. "url" => $item["thumb"]["url"],
  723. "ratio" => "16:9"
  724. ],
  725. "url" => $item["url"]
  726. ];
  727. }
  728. return $out;
  729. }
  730. public function news($get){
  731. if($get["npt"]){
  732. [$req, $ip] = $this->backend->get($get["npt"], "news");
  733. parse_str(
  734. parse_url($req, PHP_URL_QUERY),
  735. $search
  736. );
  737. if(isset($search["q"])){
  738. $search = $search["q"];
  739. }else{
  740. $search = "a"; // lol
  741. }
  742. try{
  743. $html =
  744. $this->get(
  745. $ip,
  746. "https://www.google.com" . $req,
  747. []
  748. );
  749. }catch(Exception $error){
  750. throw new Exception("Failed to get HTML");
  751. }
  752. }else{
  753. $search = $get["s"];
  754. $country = $get["country"];
  755. $nsfw = $get["nsfw"];
  756. $lang = $get["lang"];
  757. $time = $get["time"];
  758. $sort = $get["sort"];
  759. $ip = $this->backend->get_ip();
  760. $params = [
  761. "q" => $search,
  762. "tbm" => "nws",
  763. "hl" => "en",
  764. "num" => "20"
  765. ];
  766. // country
  767. if($country != "any"){
  768. $params["gl"] = $country;
  769. }
  770. // nsfw
  771. $params["safe"] = $nsfw == "yes" ? "off" : "active";
  772. // language
  773. if($lang != "any"){
  774. $params["lr"] = "lang_" . $lang;
  775. }
  776. $tbs = [];
  777. // time
  778. if($time != "any"){
  779. if($time == "a"){
  780. $tbs[] = "ar:1";
  781. }else{
  782. $tbs[] = "qdr:" . $time;
  783. }
  784. }
  785. // relevance
  786. if($sort == "date"){
  787. $tbs[] = "sbd:1";
  788. }
  789. // append tbs
  790. if(count($tbs) !== 0){
  791. $params["tbs"] =
  792. implode(",", $tbs);
  793. }
  794. $html =
  795. $this->get(
  796. $ip,
  797. "https://www.google.com/search",
  798. $params
  799. );
  800. }
  801. $json = $this->parsepage($html, "news", $search, $ip);
  802. $out = [
  803. "status" => "ok",
  804. "npt" => $json["npt"],
  805. "news" => []
  806. ];
  807. foreach($json["web"] as $item){
  808. $description = array_key_first($item["table"]);
  809. if($description !== null){
  810. $date = $item["table"][$description];
  811. }else{
  812. $date = null;
  813. }
  814. $out["news"][] = [
  815. "title" => $item["title"],
  816. "author" => $item["author"],
  817. "description" => $description,
  818. "date" => strtotime($date),
  819. "thumb" =>
  820. $item["thumb"]["url"] === null ?
  821. [
  822. "url" => null,
  823. "ratio" => null
  824. ] :
  825. [
  826. "url" => $item["thumb"]["url"],
  827. "ratio" => "16:9"
  828. ],
  829. "url" => $item["url"]
  830. ];
  831. }
  832. return $out;
  833. }
  834. private function parsepage($html, $pagetype, $search, $ip){
  835. /*
  836. $handle = fopen("scraper/google.html", "r");
  837. $html = fread($handle, filesize("scraper/google.html"));
  838. fclose($handle);
  839. */
  840. $out = [
  841. "status" => "ok",
  842. "spelling" => [
  843. "type" => "no_correction",
  844. "using" => null,
  845. "correction" => null
  846. ],
  847. "npt" => null,
  848. "answer" => [],
  849. "web" => [],
  850. "image" => [],
  851. "video" => [],
  852. "news" => [],
  853. "related" => []
  854. ];
  855. $this->parsejavascript($html);
  856. //
  857. // parse accdef's
  858. //
  859. $has_appended_accdef = false;
  860. preg_match_all(
  861. '/window\.jsl\.dh\(\'(accdef_[0-9]+)\',\'(.*)\'\);/',
  862. $html,
  863. $accdefs_regex
  864. );
  865. $accdefs = [];
  866. for($i=0; $i<count($accdefs_regex[0]); $i++){
  867. // decode UTF-16 string
  868. $answer =
  869. $this->fuckhtml
  870. ->parseJsString(
  871. $accdefs_regex[2][$i]
  872. );
  873. $this->fuckhtml->load($answer);
  874. // get description
  875. $description =
  876. $this->fuckhtml
  877. ->getElementsByClassName(
  878. $this->findstyles(
  879. [
  880. "padding" => "12px 16px 12px",
  881. ],
  882. self::is_class
  883. ),
  884. "div"
  885. )[1];
  886. // get date (rare)
  887. $date =
  888. $this->fuckhtml
  889. ->getElementsByTagName("sub");
  890. if(count($date) !== 0){
  891. $description =
  892. str_replace(
  893. $date[0]["outerHTML"],
  894. "",
  895. $description["innerHTML"]
  896. );
  897. $date =
  898. strtotime(
  899. $this->fuckhtml
  900. ->getTextContent(
  901. $date[0]
  902. )
  903. );
  904. }else{
  905. $date = null;
  906. }
  907. // get information table
  908. $table = [];
  909. $tbody =
  910. $this->fuckhtml
  911. ->getElementsByTagName("tbody");
  912. if(count($tbody) !== 0){
  913. $this->fuckhtml->load($tbody[0]);
  914. $trs =
  915. $this->fuckhtml
  916. ->getElementsByTagName("tr");
  917. foreach($trs as $tr){
  918. $this->fuckhtml->load($tr);
  919. $tds =
  920. $this->fuckhtml
  921. ->getElementsByTagName("td");
  922. if(count($tds) === 2){
  923. $table[
  924. $this->fuckhtml
  925. ->getTextContent(
  926. $tds[0]
  927. )
  928. ] =
  929. $this->fuckhtml
  930. ->getTextContent(
  931. $tds[1]
  932. );
  933. }
  934. }
  935. // load back what we had
  936. $this->fuckhtml->load($answer);
  937. }
  938. // get title & link
  939. $a =
  940. $this->fuckhtml
  941. ->getElementsByTagName("a")[0];
  942. $this->fuckhtml->load($a);
  943. $title =
  944. $this->fuckhtml
  945. ->getElementsByTagName("span");
  946. if(count($title) === 0){
  947. continue;
  948. }
  949. $accdefs[] = [
  950. "title" =>
  951. $this->titledots(
  952. $this->fuckhtml
  953. ->getTextContent(
  954. $title[0]
  955. )
  956. ),
  957. "description" =>
  958. $this->fuckhtml
  959. ->getTextContent(
  960. $description
  961. ),
  962. "url" =>
  963. $this->unshiturl(
  964. $a["attributes"]["href"]
  965. ),
  966. "date" => $date,
  967. "type" => "web",
  968. "thumb" => [
  969. "url" => null,
  970. "ratio" => null
  971. ],
  972. "sublink" => [],
  973. "table" => $table
  974. ];
  975. }
  976. $this->fuckhtml->load($html);
  977. $containers =
  978. $this->fuckhtml
  979. ->getElementsByClassName(
  980. $this->findstyles(
  981. [
  982. "background-color" => "#fff",
  983. "margin-bottom" => "10px",
  984. "-webkit-box-shadow" => "0 1px 6px rgba(32,33,36,0.28)",
  985. "border-radius" => "8px"
  986. ],
  987. self::is_class
  988. ),
  989. "div"
  990. );
  991. foreach($containers as $container){
  992. $this->fuckhtml->load($container);
  993. // detect spelling
  994. $spelling =
  995. $this->fuckhtml
  996. ->getElementsByClassName(
  997. $this->findstyles(
  998. [
  999. "font-size" => "20px",
  1000. "line-height" => "26px",
  1001. "padding-top" => "2px",
  1002. "margin-bottom" => "1px"
  1003. ],
  1004. self::is_class
  1005. ),
  1006. "div"
  1007. );
  1008. if(count($spelling) !== 0){
  1009. $a =
  1010. $this->fuckhtml
  1011. ->getElementsByTagName("a");
  1012. if(count($a) !== 0){
  1013. $scripts =
  1014. $this->fuckhtml
  1015. ->getElementsByTagName("script");
  1016. foreach($scripts as $script){
  1017. $container["innerHTML"] =
  1018. str_replace(
  1019. $script["outerHTML"],
  1020. "",
  1021. $container["innerHTML"]
  1022. );
  1023. }
  1024. $container["innerHTML"] =
  1025. $this->fuckhtml
  1026. ->getTextContent(
  1027. str_replace(
  1028. $a[0]["outerHTML"],
  1029. "",
  1030. $container["innerHTML"]
  1031. )
  1032. );
  1033. if(
  1034. preg_match(
  1035. '/^did you mean/i',
  1036. $container["innerHTML"]
  1037. )
  1038. ){
  1039. $out["spelling"] = [
  1040. "type" => "not_many",
  1041. "using" => $search,
  1042. "correction" =>
  1043. $this->fuckhtml
  1044. ->getTextContent(
  1045. $a[0]
  1046. )
  1047. ];
  1048. }
  1049. elseif(
  1050. preg_match(
  1051. '/^showing results for/i',
  1052. $container["innerHTML"]
  1053. )
  1054. ){
  1055. $out["spelling"] = [
  1056. "type" => "including",
  1057. "using" =>
  1058. $this->fuckhtml
  1059. ->getTextContent(
  1060. $a[0]
  1061. ),
  1062. "correction" => $search
  1063. ];
  1064. }
  1065. }
  1066. continue;
  1067. }
  1068. $title =
  1069. $this->fuckhtml
  1070. ->getElementsByClassName(
  1071. $this->findstyles(
  1072. [
  1073. "color" => "#1967d2",
  1074. "font-size" => "20px",
  1075. "line-height" => "26px"
  1076. ],
  1077. self::is_class
  1078. ),
  1079. "div"
  1080. );
  1081. if(count($title) !== 0){
  1082. //
  1083. // Container is a web link
  1084. //
  1085. $web = [
  1086. "title" =>
  1087. $this->titledots(
  1088. $this->fuckhtml
  1089. ->getTextContent(
  1090. $title[0]
  1091. )
  1092. ),
  1093. "description" => null,
  1094. "url" => null,
  1095. "date" => null,
  1096. "type" => "web",
  1097. "thumb" => [
  1098. "url" => null,
  1099. "ratio" => null
  1100. ],
  1101. "sublink" => [],
  1102. "table" => []
  1103. ];
  1104. // get link
  1105. $web["url"] =
  1106. $this->unshiturl(
  1107. $this->fuckhtml
  1108. ->getElementsByTagName("a")
  1109. [0]
  1110. ["attributes"]
  1111. ["href"]
  1112. );
  1113. //
  1114. // check if link contains a carousel
  1115. //
  1116. $carousels = $this->parsecarousels();
  1117. if(count($carousels) !== 0){
  1118. $first = true;
  1119. foreach($carousels as $carousel_cat){
  1120. foreach($carousel_cat as $carousel){
  1121. if($first === true){
  1122. $first = false;
  1123. }elseif($carousel["image"] !== null){
  1124. $out["image"][] = [
  1125. "title" => $carousel["title"],
  1126. "source" => [
  1127. [
  1128. "url" => $carousel["image"],
  1129. "width" => null,
  1130. "height" => null
  1131. ]
  1132. ],
  1133. "url" => $carousel["url"]
  1134. ];
  1135. }
  1136. $web["sublink"][] = [
  1137. "title" => $carousel["title"],
  1138. "date" => $carousel["date"],
  1139. "description" => $carousel["description"],
  1140. "url" => $carousel["url"]
  1141. ];
  1142. }
  1143. }
  1144. if($carousels[0][0]["image"] !== null){
  1145. $web["thumb"] = [
  1146. "url" => $carousels[0][0]["image"],
  1147. "ratio" => "16:9"
  1148. ];
  1149. }
  1150. $out["web"][] = $web;
  1151. continue;
  1152. }
  1153. //
  1154. // no carousel entries, parse as normal link
  1155. //
  1156. $this->fuckhtml->load($container);
  1157. // parse URL
  1158. $web["url"] =
  1159. $this->unshiturl(
  1160. $this->fuckhtml
  1161. ->getElementsByTagName("a")
  1162. [0]
  1163. ["attributes"]
  1164. ["href"]
  1165. );
  1166. $container = $container["innerHTML"];
  1167. $line_detect =
  1168. $this->fuckhtml
  1169. ->getElementsByClassName(
  1170. $this->findstyles(
  1171. [
  1172. "height" => "1px",
  1173. "background-color" => "#dadce0",
  1174. "margin" => "0 16px"
  1175. ],
  1176. self::is_class
  1177. ),
  1178. "div"
  1179. );
  1180. if(count($line_detect) !== 0){
  1181. // we found a line, this means we're dealing with a
  1182. // "featured snippet"
  1183. $featured = true;
  1184. $description_container =
  1185. $this->fuckhtml
  1186. ->getElementsByClassName(
  1187. $this->findstyles(
  1188. [
  1189. "white-space" => "pre-line",
  1190. "word-wrap" => "break-word"
  1191. ],
  1192. self::is_class
  1193. ),
  1194. "div"
  1195. )[1];
  1196. // get date node for it
  1197. $date =
  1198. $this->fuckhtml
  1199. ->getElementsByTagName("sub");
  1200. if(count($date) !== 0){
  1201. $web["date"] =
  1202. strtotime(
  1203. $this->fuckhtml
  1204. ->getTextContent(
  1205. $date[0]
  1206. )
  1207. );
  1208. }
  1209. }else{
  1210. // we're dealing with a normal link
  1211. $featured = false;
  1212. $description_container =
  1213. $this->fuckhtml
  1214. ->getElementsByClassName(
  1215. $this->findstyles(
  1216. [
  1217. "padding" => "12px 16px 12px"
  1218. ],
  1219. self::is_class
  1220. ),
  1221. "div"
  1222. )[1];
  1223. }
  1224. //
  1225. // Get author if we're parsing news
  1226. //
  1227. if($pagetype == "news"){
  1228. $author =
  1229. $this->fuckhtml
  1230. ->getElementsByClassName(
  1231. $this->findstyles(
  1232. [
  1233. "position" => "absolute",
  1234. "width" => "100%",
  1235. "top" => "0",
  1236. "left" => "0",
  1237. "padding-top" => "1px",
  1238. "margin-bottom" => "-1px"
  1239. ],
  1240. self::is_class
  1241. ),
  1242. "div"
  1243. );
  1244. if(count($author) !== 0){
  1245. $web["author"] =
  1246. $this->fuckhtml
  1247. ->getTextContent(
  1248. $author[0]
  1249. );
  1250. }else{
  1251. $web["author"] = null;
  1252. }
  1253. }
  1254. $description =
  1255. $description_container["innerHTML"];
  1256. $this->fuckhtml->load($description);
  1257. //
  1258. // get thumbnail before we call loadhtml again
  1259. //
  1260. $img =
  1261. $this->fuckhtml
  1262. ->getElementsByTagName("img");
  1263. if(count($img) !== 0){
  1264. $skip = true;
  1265. if(
  1266. isset($img[0]["attributes"]["alt"]) &&
  1267. stripos($img[0]["attributes"]["alt"], "Video for") !== false
  1268. ){
  1269. // is a video thumbnail
  1270. $web["thumb"]["ratio"] = "16:9";
  1271. }else{
  1272. // is a google thumbnail
  1273. $web["thumb"]["ratio"] = "1:1";
  1274. }
  1275. $web["thumb"]["url"] =
  1276. $this->getimage(
  1277. $img[0]["attributes"]["id"]
  1278. );
  1279. }else{
  1280. $skip = false;
  1281. }
  1282. //
  1283. // get sublinks
  1284. //
  1285. $links =
  1286. $this->fuckhtml
  1287. ->getElementsByTagName("a");
  1288. foreach($links as $link){
  1289. if($skip === true){
  1290. $skip = false;
  1291. continue;
  1292. }
  1293. $description =
  1294. str_replace(
  1295. $link["outerHTML"],
  1296. "",
  1297. $description
  1298. );
  1299. $sublink = [
  1300. "title" => null,
  1301. "description" => null,
  1302. "url" => null,
  1303. "date" => null
  1304. ];
  1305. $sublink["title"] =
  1306. $this->titledots(
  1307. $this->fuckhtml
  1308. ->getTextContent(
  1309. $link
  1310. )
  1311. );
  1312. $sublink["url"] =
  1313. $this->unshiturl(
  1314. $link
  1315. ["attributes"]
  1316. ["href"]
  1317. );
  1318. if(parse_url($sublink["url"], PHP_URL_HOST) !== null){
  1319. $web["sublink"][] = $sublink;
  1320. }
  1321. }
  1322. //
  1323. // Parse spans in description
  1324. //
  1325. $this->fuckhtml->load($description);
  1326. if($featured === false){
  1327. $levels =
  1328. $this->fuckhtml
  1329. ->getElementsByClassName(
  1330. $this->findstyles(
  1331. [
  1332. "padding-bottom" => "8px"
  1333. ],
  1334. self::is_class
  1335. ),
  1336. "div"
  1337. );
  1338. // oh my god yes, fucking great, sometimes there are NO levels
  1339. // hahahahahhahahahahahahahahahhahaa
  1340. if(count($levels) === 0){
  1341. $levels = [$description];
  1342. }
  1343. foreach($levels as $level){
  1344. $this->fuckhtml->load($level);
  1345. $spans =
  1346. $this->fuckhtml
  1347. ->getElementsByTagName(
  1348. "span"
  1349. );
  1350. $is_rating = -1;
  1351. foreach($spans as $span){
  1352. $innertext =
  1353. trim(
  1354. $this->fuckhtml
  1355. ->getTextContent(
  1356. $span
  1357. ),
  1358. " ·."
  1359. );
  1360. if($innertext == ""){ continue; }
  1361. if(
  1362. strtolower($innertext)
  1363. == "rating"
  1364. ){
  1365. $is_rating = 0;
  1366. // clean up before we go
  1367. $description =
  1368. str_replace(
  1369. $span["outerHTML"],
  1370. "",
  1371. $description
  1372. );
  1373. continue;
  1374. }
  1375. //
  1376. // Parse rating object
  1377. //
  1378. if($is_rating >= 0){
  1379. // clean up description
  1380. $description =
  1381. str_replace(
  1382. $span["outerHTML"],
  1383. "",
  1384. $description
  1385. );
  1386. if($span["level"] !== 1){ continue; }
  1387. $is_rating++;
  1388. // 10/10 (123)
  1389. if($is_rating === 1){
  1390. $innertext = explode(" ", $innertext, 2);
  1391. $web["table"]["Rating"] = $innertext[0];
  1392. if(count($innertext) === 2){
  1393. $web["table"]["Hits"] =
  1394. trim(
  1395. str_replace(
  1396. [
  1397. "(",
  1398. ")"
  1399. ],
  1400. "",
  1401. $innertext[1]
  1402. )
  1403. );
  1404. if($web["table"]["Hits"] == ""){
  1405. unset($web["table"]["Hits"]);
  1406. }
  1407. }
  1408. continue;
  1409. }
  1410. // US$4.99
  1411. // MYR 50.00
  1412. // $38.34
  1413. // JP¥6,480
  1414. // Reviewed by your mom
  1415. if($is_rating === 2){
  1416. if(
  1417. preg_match(
  1418. '/^Review by (.+)/',
  1419. $innertext,
  1420. $match
  1421. )
  1422. ){
  1423. $web["table"]["Author"] = $match[1];
  1424. continue;
  1425. }
  1426. $web["table"]["Price"] = $innertext;
  1427. continue;
  1428. }
  1429. // Android / In stock
  1430. if($is_rating === 3){
  1431. $web["table"]["Support"] = $innertext;
  1432. continue;
  1433. }
  1434. // ignore the rest
  1435. continue;
  1436. }
  1437. //
  1438. // Parse standalone text
  1439. //
  1440. // If we reach this point:
  1441. // 1. Ratings have been parsed
  1442. // 2. We're parsing a WEB link, not some shitty piece of shit
  1443. // check for date
  1444. // if span has no text before it, assume it's a date
  1445. $desc_split =
  1446. explode(
  1447. $span["outerHTML"],
  1448. $description,
  1449. 2
  1450. );
  1451. if(
  1452. $this->fuckhtml
  1453. ->getTextContent(
  1454. $desc_split[0]
  1455. ) == ""
  1456. ){
  1457. // has no text before
  1458. $date = strtotime($innertext);
  1459. if($date){
  1460. $web["date"] = $date;
  1461. }
  1462. // cleanup
  1463. $description =
  1464. str_replace(
  1465. $span["outerHTML"],
  1466. "",
  1467. $description
  1468. );
  1469. continue;
  1470. }
  1471. // Ready to parse table
  1472. if(count($desc_split) === 2){
  1473. $this->fuckhtml->load($desc_split[1]);
  1474. $web["table"][
  1475. $this->fuckhtml
  1476. ->getTextContent(
  1477. trim($desc_split[0], ": ")
  1478. )
  1479. ] = $innertext;
  1480. // cleanup
  1481. $description =
  1482. str_replace(
  1483. $desc_split[0] . $span["outerHTML"],
  1484. "",
  1485. $description
  1486. );
  1487. }
  1488. }
  1489. }
  1490. }
  1491. $web["description"] =
  1492. trim(
  1493. $this->fuckhtml
  1494. ->getTextContent(
  1495. $description
  1496. ),
  1497. " ·."
  1498. );
  1499. if($web["description"] == ""){
  1500. $web["description"] = null;
  1501. }
  1502. $out["web"][] = $web;
  1503. continue;
  1504. }
  1505. //
  1506. // Detect wikipedia shit
  1507. //
  1508. $wiki_title =
  1509. $this->fuckhtml
  1510. ->getElementsByTagName("h3");
  1511. if(count($wiki_title) !== 0){
  1512. $description_after = [];
  1513. $description = [];
  1514. $table = [];
  1515. $sublink = [];
  1516. $as =
  1517. $this->fuckhtml
  1518. ->getElementsByTagName("a");
  1519. foreach($as as $a){
  1520. if(
  1521. isset($a["attributes"]["href"]) &&
  1522. parse_url($a["attributes"]["href"], PHP_URL_HOST) == "maps.google.com"
  1523. ){
  1524. // detected maps embed, ignore
  1525. continue 2;
  1526. }
  1527. }
  1528. // get carousels and remove them from container for image grepper
  1529. $carousels = $this->parsecarousels($container["innerHTML"]);
  1530. $this->fuckhtml->load($container);
  1531. // add images to image tab, if applicable
  1532. for($i=0; $i<count($carousels); $i++){
  1533. foreach($carousels[$i] as $item){
  1534. if(
  1535. $item["url"] !== null &&
  1536. $item["ref"] !== null &&
  1537. $item["image"] !== null &&
  1538. $item["title"] !== null
  1539. ){
  1540. $out["image"][] = [
  1541. "title" => $item["title"],
  1542. "source" => [
  1543. [
  1544. "url" => $item["url"],
  1545. "width" => $item["image_width"],
  1546. "height" => $item["image_height"]
  1547. ],
  1548. [
  1549. "url" => $item["image"],
  1550. "width" => $item["thumb_width"],
  1551. "height" => $item["thumb_height"]
  1552. ]
  1553. ],
  1554. "url" => $item["ref"]
  1555. ];
  1556. unset($carousels[$i]);
  1557. }
  1558. }
  1559. }
  1560. $carousels = array_values($carousels);
  1561. // interpret remaining carousels as title + carousel
  1562. $titles =
  1563. $this->fuckhtml
  1564. ->getElementsByClassName(
  1565. $this->findstyles(
  1566. [
  1567. "font-weight" => "700",
  1568. "letter-spacing" => "0.75px",
  1569. "text-transform" => "uppercase"
  1570. ],
  1571. self::is_class
  1572. )
  1573. );
  1574. for($i=0; $i<count($titles); $i++){
  1575. if(!isset($carousels[$i])){
  1576. break;
  1577. }
  1578. $description_after[] = [
  1579. "type" => "title",
  1580. "value" =>
  1581. $this->fuckhtml
  1582. ->getTextContent(
  1583. $titles[$i]
  1584. )
  1585. ];
  1586. foreach($carousels[$i] as $carousel){
  1587. $description_after[] = [
  1588. "type" => "link",
  1589. "url" => "web?s=" . urlencode($carousel["description"]) . "&scraper=google",
  1590. "value" => $carousel["description"]
  1591. ];
  1592. if($carousel["subtext"] !== null){
  1593. $description_after[] = [
  1594. "type" => "quote",
  1595. "value" => $carousel["subtext"]
  1596. ];
  1597. }
  1598. $description_after[] = [
  1599. "type" => "image",
  1600. "url" => $carousel["image"]
  1601. ];
  1602. }
  1603. }
  1604. $categories =
  1605. $this->fuckhtml
  1606. ->getElementsByClassName(
  1607. $this->findstyles(
  1608. [
  1609. "padding" => "12px 16px 12px"
  1610. ],
  1611. self::is_class
  1612. )
  1613. );
  1614. $image =
  1615. $this->fuckhtml
  1616. ->getElementsByTagName("img");
  1617. if(count($image) !== 0){
  1618. $image = $this->getimage($image[0]["attributes"]["id"]);
  1619. }else{
  1620. $image = null;
  1621. }
  1622. $url = null;
  1623. for($i=0; $i<count($categories); $i++){
  1624. $this->fuckhtml->load($categories[$i]);
  1625. if($i === 0){
  1626. // first node. this should be the header with the small
  1627. // information snippet
  1628. $url =
  1629. $this->fuckhtml
  1630. ->getElementsByTagName("a");
  1631. if(count($url) !== 0){
  1632. $url =
  1633. $this->unshiturl(
  1634. $url[0]["attributes"]["href"]
  1635. );
  1636. if(parse_url($url, PHP_URL_HOST) == "encrypted-tbn0.gstatic.com"){
  1637. $image = $url;
  1638. $url = null;
  1639. }
  1640. }else{
  1641. $url = null;
  1642. }
  1643. $categories[$i]["innerHTML"] =
  1644. str_replace(
  1645. $wiki_title[0]["outerHTML"],
  1646. "",
  1647. $categories[$i]["innerHTML"]
  1648. );
  1649. $subtext =
  1650. $this->fuckhtml
  1651. ->getTextContent(
  1652. $categories[$i]["innerHTML"]
  1653. );
  1654. if(strlen($subtext) !== 0){
  1655. $description[] = [
  1656. "type" => "quote",
  1657. "value" =>
  1658. $this->fuckhtml
  1659. ->getTextContent(
  1660. $categories[$i]["innerHTML"]
  1661. )
  1662. ];
  1663. }
  1664. // detect audio file
  1665. $audio =
  1666. $this->fuckhtml
  1667. ->getElementsByTagName("audio");
  1668. if(count($audio) !== 0){
  1669. $description[] = [
  1670. "type" => "audio",
  1671. "url" =>
  1672. $this->fuckhtml
  1673. ->getTextContent(
  1674. $audio[0]["attributes"]["src"]
  1675. )
  1676. ];
  1677. }
  1678. }else{
  1679. // check for separator elements IN THERE
  1680. $separators =
  1681. $this->fuckhtml
  1682. ->getElementsByClassName(
  1683. $this->findstyles(
  1684. [
  1685. "white-space" => "pre-line",
  1686. "word-wrap" => "break-word"
  1687. ],
  1688. self::is_class
  1689. ),
  1690. "div"
  1691. );
  1692. // detect container type
  1693. foreach($separators as $separator){
  1694. $this->fuckhtml->load($separator);
  1695. // ignore wrong levels
  1696. if($separator["level"] !== 2){
  1697. continue;
  1698. }
  1699. //
  1700. // Detect word definition
  1701. //
  1702. $wordwraps =
  1703. $this->fuckhtml
  1704. ->getElementsByClassName(
  1705. $this->findstyles(
  1706. [
  1707. "padding-bottom" => "12px"
  1708. ],
  1709. self::is_class
  1710. ),
  1711. "div"
  1712. );
  1713. if(count($wordwraps) !== 0){
  1714. foreach($wordwraps as $word){
  1715. $this->fuckhtml->load($word);
  1716. // detect title
  1717. $span =
  1718. $this->fuckhtml
  1719. ->getElementsByTagName(
  1720. "span"
  1721. );
  1722. if(
  1723. count($span) === 1 &&
  1724. $this->fuckhtml
  1725. ->getTextContent(
  1726. str_replace(
  1727. $span[0]["outerHTML"],
  1728. "",
  1729. $word["innerHTML"]
  1730. )
  1731. ) == ""
  1732. ){
  1733. $description[] = [
  1734. "type" => "title",
  1735. "value" =>
  1736. $this->fuckhtml
  1737. ->getTextContent(
  1738. $span[0]
  1739. )
  1740. ];
  1741. continue;
  1742. }
  1743. // detect list element
  1744. $lists =
  1745. $this->fuckhtml
  1746. ->getElementsByTagName("ol");
  1747. if(count($lists) !== 0){
  1748. foreach($lists as $list){
  1749. $this->fuckhtml->load($list);
  1750. $items =
  1751. $this->fuckhtml
  1752. ->getElementsByTagName("li");
  1753. $w = 0;
  1754. foreach($items as $item){
  1755. $w++;
  1756. $this->fuckhtml->load($item);
  1757. // get subnodes
  1758. $subnodes =
  1759. $this->fuckhtml
  1760. ->getElementsByClassName(
  1761. $this->findstyles(
  1762. [
  1763. "white-space" => "pre-line",
  1764. "word-wrap" => "break-word"
  1765. ],
  1766. self::is_class
  1767. ),
  1768. "div"
  1769. );
  1770. foreach($subnodes as $subnode){
  1771. $this->fuckhtml->load($subnode);
  1772. $spans =
  1773. $this->fuckhtml
  1774. ->getElementsByTagName("span");
  1775. if(count($spans) !== 0){
  1776. // append quote
  1777. $description[] = [
  1778. "type" => "quote",
  1779. "value" =>
  1780. $this->fuckhtml
  1781. ->getTextContent(
  1782. $subnode
  1783. )
  1784. ];
  1785. }else{
  1786. // append text
  1787. $description[] = [
  1788. "type" => "text",
  1789. "value" =>
  1790. $w . ". " .
  1791. $this->fuckhtml
  1792. ->getTextContent(
  1793. $subnode
  1794. )
  1795. ];
  1796. }
  1797. }
  1798. }
  1799. }
  1800. }else{
  1801. // parse without list
  1802. // get subnodes
  1803. $subnodes =
  1804. $this->fuckhtml
  1805. ->getElementsByClassName(
  1806. $this->findstyles(
  1807. [
  1808. "white-space" => "pre-line",
  1809. "word-wrap" => "break-word"
  1810. ],
  1811. self::is_class
  1812. ),
  1813. "div"
  1814. );
  1815. foreach($subnodes as $subnode){
  1816. $this->fuckhtml->load($subnode);
  1817. $spans =
  1818. $this->fuckhtml
  1819. ->getElementsByTagName("span");
  1820. if(count($spans) !== 0){
  1821. // append quote
  1822. $description[] = [
  1823. "type" => "quote",
  1824. "value" =>
  1825. $this->fuckhtml
  1826. ->getTextContent(
  1827. $subnode
  1828. )
  1829. ];
  1830. }else{
  1831. // append text
  1832. $description[] = [
  1833. "type" => "text",
  1834. "value" =>
  1835. $this->fuckhtml
  1836. ->getTextContent(
  1837. $subnode
  1838. )
  1839. ];
  1840. }
  1841. }
  1842. }
  1843. }
  1844. }else{
  1845. //
  1846. // Parse table
  1847. //
  1848. $spans =
  1849. $this->fuckhtml
  1850. ->getElementsByTagName("span");
  1851. foreach($spans as $span){
  1852. if(!isset($span["attributes"]["class"])){
  1853. // found table
  1854. $row =
  1855. explode(
  1856. ":",
  1857. $this->fuckhtml
  1858. ->getTextContent(
  1859. $separator
  1860. ),
  1861. 2
  1862. );
  1863. if(count($row) === 2){
  1864. $table[rtrim($row[0])] =
  1865. ltrim($row[1]);
  1866. }
  1867. continue 2;
  1868. }
  1869. }
  1870. //
  1871. // Parse normal description
  1872. //
  1873. $links_rem =
  1874. $this->fuckhtml
  1875. ->getElementsByTagName("a");
  1876. foreach($links_rem as $rem){
  1877. $separator["innerHTML"] =
  1878. str_replace(
  1879. $rem["outerHTML"],
  1880. "",
  1881. $separator["innerHTML"]
  1882. );
  1883. }
  1884. $description[] = [
  1885. "type" => "text",
  1886. "value" =>
  1887. rtrim(
  1888. $this->fuckhtml
  1889. ->getTextContent(
  1890. $separator
  1891. ),
  1892. " .,"
  1893. )
  1894. ];
  1895. }
  1896. }
  1897. }
  1898. // detect huge buttons
  1899. $buttons =
  1900. $this->fuckhtml
  1901. ->getElementsByClassName(
  1902. $this->findstyles(
  1903. [
  1904. "display" => "table-cell",
  1905. "vertical-align" => "middle",
  1906. "height" => "52px",
  1907. "text-align" => "center"
  1908. ],
  1909. self::is_class
  1910. ),
  1911. "a"
  1912. );
  1913. if(count($buttons) !== 0){
  1914. foreach($buttons as $button){
  1915. if(isset($button["attributes"]["href"])){
  1916. $sublink[
  1917. $this->fuckhtml
  1918. ->getTextContent(
  1919. $button
  1920. )
  1921. ] =
  1922. $this->unshiturl(
  1923. $button["attributes"]["href"]
  1924. );
  1925. }
  1926. }
  1927. }
  1928. }
  1929. // append description_after (contains carousel info)
  1930. $description = array_merge(
  1931. $description,
  1932. $description_after
  1933. );
  1934. $out["answer"][] = [
  1935. "title" =>
  1936. $this->fuckhtml
  1937. ->getTextContent(
  1938. $wiki_title[0]
  1939. ),
  1940. "description" => $description,
  1941. "url" => $url,
  1942. "thumb" => $image,
  1943. "table" => $table,
  1944. "sublink" => $sublink
  1945. ];
  1946. continue;
  1947. }
  1948. //
  1949. // Detect related searches containers
  1950. //
  1951. $container_title =
  1952. $this->fuckhtml
  1953. ->getElementsByClassName(
  1954. $this->findstyles(
  1955. [
  1956. "font-weight" => "bold",
  1957. "font-size" => "16px",
  1958. "color" => "#000",
  1959. "margin" => "0",
  1960. "padding" => "12px 16px 0 16px"
  1961. ],
  1962. self::is_class
  1963. ),
  1964. "div"
  1965. );
  1966. if(count($container_title) !== 0){
  1967. // get carousel entries
  1968. $carousels = $this->parsecarousels($container["innerHTML"]);
  1969. $this->fuckhtml->load($container);
  1970. foreach($carousels as $carousel){
  1971. foreach($carousel as $item){
  1972. if($item["url"] !== null){
  1973. $out["related"][] = $item["url"];
  1974. }
  1975. }
  1976. }
  1977. $container_title =
  1978. strtolower(
  1979. $this->fuckhtml
  1980. ->getTextContent(
  1981. $container_title[0]
  1982. )
  1983. );
  1984. switch($container_title){
  1985. case "related searches":
  1986. case "people also search for":
  1987. //
  1988. // Parse related searches
  1989. //
  1990. $as =
  1991. $this->fuckhtml
  1992. ->getElementsByTagName("a");
  1993. foreach($as as $a){
  1994. $out["related"][] =
  1995. $this->fuckhtml
  1996. ->getTextContent($a);
  1997. }
  1998. break;
  1999. case "people also ask":
  2000. // get related queries
  2001. $divs =
  2002. $this->fuckhtml
  2003. ->getElementsByTagName("div");
  2004. foreach($divs as $div){
  2005. // add accdef's here
  2006. if($has_appended_accdef === false){
  2007. $out["web"] = array_merge($out["web"], $accdefs);
  2008. $has_appended_accdef = true;
  2009. }
  2010. // add accdef's questions
  2011. if(isset($div["attributes"]["role"])){
  2012. $out["related"][] =
  2013. $this->fuckhtml
  2014. ->getTextContent($div);
  2015. continue;
  2016. }
  2017. }
  2018. break;
  2019. }
  2020. continue;
  2021. }
  2022. //
  2023. // Parse news
  2024. //
  2025. $title =
  2026. $this->fuckhtml
  2027. ->getElementsByClassName(
  2028. $this->findstyles(
  2029. [
  2030. "font-size" => "16px",
  2031. "line-height" => "20px",
  2032. "font-weight" => "400"
  2033. ],
  2034. self::is_class
  2035. ),
  2036. "div"
  2037. );
  2038. if(count($title) !== 0){
  2039. $carousels = $this->parsecarousels();
  2040. $this->fuckhtml->load($container);
  2041. if(count($carousels) === 0){
  2042. // no carousels found
  2043. continue;
  2044. }
  2045. $title =
  2046. strtolower(
  2047. $this->fuckhtml
  2048. ->getTextContent(
  2049. $title[0]
  2050. )
  2051. );
  2052. if(
  2053. preg_match(
  2054. '/^latest from|^top stories/',
  2055. $title
  2056. )
  2057. ){
  2058. // Found news article
  2059. foreach($carousels[0] as $carousel){
  2060. if($carousel["image"] !== null){
  2061. $thumb = [
  2062. "url" => $carousel["image"],
  2063. "ratio" => "16:9"
  2064. ];
  2065. }else{
  2066. $thumb = [
  2067. "url" => null,
  2068. "ratio" => null
  2069. ];
  2070. }
  2071. $out["news"][] = [
  2072. "title" => $carousel["title"],
  2073. "description" => $carousel["description"],
  2074. "date" => $carousel["date"],
  2075. "thumb" => $thumb,
  2076. "url" => $carousel["url"]
  2077. ];
  2078. }
  2079. }
  2080. elseif(
  2081. $title == "images"
  2082. ){
  2083. foreach($carousels as $carousel){
  2084. foreach($carousel as $item){
  2085. $out["image"][] = [
  2086. "title" => $item["title"],
  2087. "source" => [
  2088. [
  2089. "url" => $item["url"],
  2090. "width" => $item["image_width"],
  2091. "height" => $item["image_height"]
  2092. ],
  2093. [
  2094. "url" => $item["image"],
  2095. "width" => $item["thumb_width"],
  2096. "height" => $item["thumb_height"]
  2097. ]
  2098. ],
  2099. "url" => $item["ref"]
  2100. ];
  2101. }
  2102. }
  2103. }
  2104. continue;
  2105. }
  2106. //
  2107. // Detect nodes with only text + links
  2108. //
  2109. // ignore elements with <style> tags
  2110. $style =
  2111. $this->fuckhtml
  2112. ->getElementsByTagName("style");
  2113. if(count($style) !== 0){
  2114. continue;
  2115. }
  2116. $as =
  2117. $this->fuckhtml
  2118. ->getElementsByTagName("a");
  2119. $description = [];
  2120. foreach($as as $a){
  2121. //
  2122. // Detect next page
  2123. //
  2124. if(
  2125. isset($a["attributes"]["aria-label"]) &&
  2126. strtolower($a["attributes"]["aria-label"]) == "next page"
  2127. ){
  2128. $out["npt"] =
  2129. $this->backend->store(
  2130. $this->fuckhtml
  2131. ->getTextContent(
  2132. $a["attributes"]["href"]
  2133. ),
  2134. $pagetype,
  2135. $ip
  2136. );
  2137. continue 2;
  2138. }
  2139. //
  2140. // Parse as text node
  2141. //
  2142. $container["innerHTML"] =
  2143. explode(
  2144. $a["outerHTML"],
  2145. $container["innerHTML"],
  2146. 2
  2147. );
  2148. $before =
  2149. $this->fuckhtml
  2150. ->getTextContent(
  2151. $container["innerHTML"][0],
  2152. false,
  2153. false
  2154. );
  2155. // set after
  2156. if(count($container["innerHTML"]) === 2){
  2157. $container["innerHTML"] =
  2158. $container["innerHTML"][1];
  2159. }else{
  2160. $container["innerHTML"] = "";
  2161. }
  2162. if($before != ""){
  2163. $description[] = [
  2164. "type" => "text",
  2165. "value" => $before
  2166. ];
  2167. }
  2168. // add link
  2169. $description[] = [
  2170. "type" => "link",
  2171. "url" =>
  2172. $this->unshiturl(
  2173. $a["attributes"]
  2174. ["href"]
  2175. ),
  2176. "value" =>
  2177. $this->fuckhtml
  2178. ->getTextContent(
  2179. $a
  2180. )
  2181. ];
  2182. }
  2183. if($container["innerHTML"] != ""){
  2184. $description[] = [
  2185. "type" => "text",
  2186. "value" =>
  2187. $this->fuckhtml
  2188. ->getTextContent(
  2189. $container["innerHTML"]
  2190. )
  2191. ];
  2192. }
  2193. $out["answer"][] = [
  2194. "title" => "Notice",
  2195. "description" => $description,
  2196. "url" => null,
  2197. "thumb" => null,
  2198. "table" => [],
  2199. "sublink" => []
  2200. ];
  2201. }
  2202. //
  2203. // remove duplicate web links cause instant answers
  2204. // sometimes contains duplicates
  2205. //
  2206. $c = count($out["web"]);
  2207. $links = [];
  2208. for($i=0; $i<$c; $i++){
  2209. foreach($links as $link){
  2210. if($out["web"][$i]["url"] == $link){
  2211. unset($out["web"][$i]);
  2212. continue 2;
  2213. }
  2214. }
  2215. $links[] = $out["web"][$i]["url"];
  2216. }
  2217. $out["web"] = array_values($out["web"]);
  2218. return $out;
  2219. }
  2220. public function image($get){
  2221. // generate parameters
  2222. if($get["npt"]){
  2223. [$params, $proxy] =
  2224. $this->backend->get(
  2225. $get["npt"],
  2226. "images"
  2227. );
  2228. $params = json_decode($params, true);
  2229. }else{
  2230. $search = $get["s"];
  2231. if(strlen($search) === 0){
  2232. throw new Exception("Search term is empty!");
  2233. }
  2234. $proxy = $this->backend->get_ip();
  2235. $country = $get["country"];
  2236. $nsfw = $get["nsfw"];
  2237. $lang = $get["lang"];
  2238. $time = $get["time"];
  2239. $size = $get["size"];
  2240. $ratio = $get["ratio"];
  2241. $color = $get["color"];
  2242. $type = $get["type"];
  2243. $format = $get["format"];
  2244. $rights = $get["rights"];
  2245. $params = [
  2246. "q" => $search,
  2247. "tbm" => "isch"
  2248. ];
  2249. // country
  2250. if($country != "any"){
  2251. $params["gl"] = $country;
  2252. }
  2253. // nsfw
  2254. $params["safe"] = $nsfw == "yes" ? "off" : "active";
  2255. // language
  2256. if($lang != "any"){
  2257. $params["lr"] = "lang_" . $lang;
  2258. }
  2259. $tbs = [];
  2260. // time
  2261. if($time != "any"){
  2262. $tbs[] = "qrd:" . $time;
  2263. }
  2264. // size
  2265. if($size != "any"){
  2266. if(
  2267. in_array(
  2268. $size,
  2269. ["l", "s", "i"]
  2270. )
  2271. ){
  2272. $tbs[] = "isz:" . $size;
  2273. }else{
  2274. $tbs[] = "tbz:lt";
  2275. $tbs[] = "islt:" . $size;
  2276. }
  2277. }
  2278. // ratio
  2279. if($ratio != "any"){
  2280. $tbs[] = "iar:" . $ratio;
  2281. }
  2282. // color
  2283. if($color != "any"){
  2284. if(
  2285. in_array(
  2286. $color,
  2287. ["color", "gray", "trans"]
  2288. )
  2289. ){
  2290. $tbs[] = "ic:" . $color;
  2291. }else{
  2292. $tbs[] = "ic:specific";
  2293. $tbs[] = "isc:" . $color;
  2294. }
  2295. }
  2296. // type
  2297. if($type != "any"){
  2298. $tbs[] = "itp:" . $type;
  2299. }
  2300. // format
  2301. if($format != "any"){
  2302. $tbs[] = "ift:" . $format;
  2303. }
  2304. // rights
  2305. if($rights != "any"){
  2306. $tbs[] = "il:" . $rights;
  2307. }
  2308. // append tbs
  2309. if(count($tbs) !== 0){
  2310. $params["tbs"] =
  2311. implode(",", $tbs);
  2312. }
  2313. }
  2314. /*
  2315. $handle = fopen("scraper/google-img.html", "r");
  2316. $html = fread($handle, filesize("scraper/google-img.html"));
  2317. fclose($handle);*/
  2318. // scrape images
  2319. try{
  2320. $html =
  2321. $this->get(
  2322. $proxy,
  2323. "https://www.google.com/search",
  2324. $params
  2325. );
  2326. }catch(Exception $error){
  2327. throw new Exception("Failed to get search page");
  2328. }
  2329. $this->fuckhtml->load($html);
  2330. $out = [
  2331. "status" => "ok",
  2332. "npt" => null,
  2333. "image" => []
  2334. ];
  2335. $images =
  2336. $this->fuckhtml
  2337. ->getElementsByClassName(
  2338. "islrtb isv-r",
  2339. "div"
  2340. );
  2341. foreach($images as $image){
  2342. $this->fuckhtml->load($image);
  2343. $img =
  2344. $this->fuckhtml
  2345. ->getElementsByTagName("img")[0];
  2346. $og_width = (int)$image["attributes"]["data-ow"];
  2347. $og_height = (int)$image["attributes"]["data-oh"];
  2348. $thumb_width = (int)$image["attributes"]["data-tw"];
  2349. $ratio = $og_width / $og_height;
  2350. if(isset($img["attributes"]["data-src"])){
  2351. $src = &$img["attributes"]["data-src"];
  2352. }else{
  2353. $src = &$img["attributes"]["src"];
  2354. }
  2355. $thumb_height = floor($thumb_width / $ratio);
  2356. $out["image"][] = [
  2357. "title" =>
  2358. $this->titledots(
  2359. $this->fuckhtml
  2360. ->getTextContent(
  2361. $image["attributes"]["data-pt"]
  2362. )
  2363. ),
  2364. "source" => [
  2365. [
  2366. "url" =>
  2367. $this->fuckhtml
  2368. ->getTextContent(
  2369. $image["attributes"]["data-ou"]
  2370. ),
  2371. "width" => $og_width,
  2372. "height" => $og_height
  2373. ],
  2374. [
  2375. "url" =>
  2376. $this->fuckhtml
  2377. ->getTextContent(
  2378. $src
  2379. ),
  2380. "width" => $thumb_width,
  2381. "height" => $thumb_height
  2382. ]
  2383. ],
  2384. "url" =>
  2385. $this->fuckhtml
  2386. ->getTextContent(
  2387. $image["attributes"]["data-ru"]
  2388. )
  2389. ];
  2390. }
  2391. // get next page
  2392. // https://www.google.com/search
  2393. // ?q=higurashi
  2394. // &tbm=isch
  2395. // &async=_id%3Aislrg_c%2C_fmt%3Ahtml
  2396. // &asearch=ichunklite
  2397. // &ved=0ahUKEwidjYXJqJSAAxWrElkFHZ07CDwQtDIIQygA
  2398. if(count($out["image"]) !== 100){
  2399. // no more results
  2400. return $out;
  2401. }
  2402. if($get["npt"]){
  2403. // update nextpage information
  2404. $params["start"] = (int)$params["start"] + count($out["image"]);
  2405. $params["ijn"] = (int)$params["ijn"] + 1;
  2406. $out["npt"] =
  2407. $this->backend->store(
  2408. json_encode($params),
  2409. "images",
  2410. $proxy
  2411. );
  2412. }else{
  2413. // scrape nextpage information
  2414. $this->fuckhtml->load($html);
  2415. $ved =
  2416. $this->fuckhtml
  2417. ->getElementById("islrg", "div");
  2418. if($ved){
  2419. $ved =
  2420. $this->fuckhtml
  2421. ->getTextContent(
  2422. $ved["attributes"]["data-ved"]
  2423. );
  2424. // &vet=1{$ved}..i (10ahUKEwidjYXJqJSAAxWrElkFHZ07CDwQtDIIQygA..i)
  2425. /*
  2426. These 2 are handled by us
  2427. start = start + number of results
  2428. ijn = current page number
  2429. */
  2430. // &start=100
  2431. // &ijn=1
  2432. // &imgvl=CAEY7gQgBSj3Aji8VTjXVUC4AUC3AUgAYNdV
  2433. preg_match(
  2434. '/var e=\'([A-z0-9]+)\';/',
  2435. $html,
  2436. $imgvl
  2437. );
  2438. if(isset($imgvl[1])){
  2439. $imgvl = $imgvl[1];
  2440. $params["async"] = "_id:islrg_c,_fmt:html";
  2441. $params["asearch"] = "ichunklite";
  2442. $params["ved"] = $ved;
  2443. $params["vet"] = "1" . $ved . "..i";
  2444. $params["start"] = 100;
  2445. $params["ijn"] = 1;
  2446. $params["imgvl"] = $imgvl;
  2447. $out["npt"] =
  2448. $this->backend->store(
  2449. json_encode($params),
  2450. "images",
  2451. $proxy
  2452. );
  2453. }
  2454. }
  2455. }
  2456. return $out;
  2457. }
  2458. private function hms2int($time){
  2459. $parts = explode(":", $time, 3);
  2460. $time = 0;
  2461. if(count($parts) === 3){
  2462. // hours
  2463. $time = $time + ((int)$parts[0] * 3600);
  2464. array_shift($parts);
  2465. }
  2466. if(count($parts) === 2){
  2467. // minutes
  2468. $time = $time + ((int)$parts[0] * 60);
  2469. array_shift($parts);
  2470. }
  2471. // seconds
  2472. $time = $time + (int)$parts[0];
  2473. return $time;
  2474. }
  2475. private function parsejavascript($html){
  2476. $this->fuckhtml->load($html);
  2477. $styles =
  2478. $this->fuckhtml
  2479. ->getElementsByTagName("style");
  2480. $this->computedstyle = [];
  2481. $this->ask = [];
  2482. foreach($styles as $style){
  2483. $this->computedstyle =
  2484. array_merge(
  2485. $this->computedstyle,
  2486. $this->parsestyles($style["innerHTML"])
  2487. );
  2488. }
  2489. // get images in javascript var
  2490. preg_match(
  2491. '/google\.ldi=({[^}]+})/',
  2492. $html,
  2493. $this->js_image
  2494. );
  2495. if(count($this->js_image) !== 0){
  2496. $this->js_image = json_decode($this->js_image[1], true);
  2497. }else{
  2498. $this->js_image = [];
  2499. }
  2500. // additional js_images present in <script> tags
  2501. // ugh i fucking hate you
  2502. $scripts =
  2503. $this->fuckhtml
  2504. ->getElementsByTagName("script");
  2505. foreach($scripts as $script){
  2506. if(!isset($script["innerHTML"])){
  2507. continue;
  2508. }
  2509. preg_match_all(
  2510. '/var s=\'(data:image[^\']+)\';var i=\[(\'[^\;]*\')];/',
  2511. $script["innerHTML"],
  2512. $image_grep
  2513. );
  2514. if(count($image_grep[0]) !== 0){
  2515. $items = explode(",", $image_grep[2][0]);
  2516. $value =
  2517. $this->fuckhtml
  2518. ->getTextContent(
  2519. $image_grep[1][0]
  2520. );
  2521. foreach($items as $item){
  2522. $this->js_image[trim($item, "' ")] = $value;
  2523. }
  2524. }
  2525. // even more javascript crap
  2526. // "People also ask" node is loaded trough javascript
  2527. preg_match_all(
  2528. '/window\.jsl\.dh\(\'([^\']+)\',\'(.+)\'\);/',
  2529. $script["innerHTML"],
  2530. $ask_grep
  2531. );
  2532. for($i=0; $i<count($ask_grep[0]); $i++){
  2533. $this->ask[trim($ask_grep[1][$i])] =
  2534. $this->fuckhtml->parseJsString(
  2535. $ask_grep[2][$i]
  2536. );
  2537. }
  2538. }
  2539. }
  2540. private function findstyles($rules, $is){
  2541. ksort($rules);
  2542. foreach($this->computedstyle as $stylename => $styles){
  2543. if($styles == $rules){
  2544. preg_match(
  2545. '/\\' . $is . '([^ .]+)/',
  2546. $stylename,
  2547. $out
  2548. );
  2549. if(count($out) === 2){
  2550. return $out[1];
  2551. }
  2552. return false;
  2553. }
  2554. }
  2555. return false;
  2556. }
  2557. private function parsestyles($style){
  2558. // get style tags
  2559. preg_match_all(
  2560. '/([^{]+){([^}]+)}/',
  2561. $style,
  2562. $tags_regex
  2563. );
  2564. $tags = [];
  2565. for($i=0; $i<count($tags_regex[0]); $i++){
  2566. $tagnames = explode(",", trim($tags_regex[1][$i]));
  2567. foreach($tagnames as $tagname){
  2568. $tagname = trim($tagname);
  2569. if(!isset($tags[$tagname])){
  2570. $tags[$tagname] = [];
  2571. }
  2572. $values = explode(";", $tags_regex[2][$i]);
  2573. foreach($values as $value){
  2574. $value = explode(":", $value, 2);
  2575. if(count($value) !== 2){
  2576. continue;
  2577. }
  2578. $tags[$tagname][trim($value[0])] =
  2579. trim($value[1]);
  2580. }
  2581. }
  2582. }
  2583. foreach($tags as &$value){
  2584. ksort($value);
  2585. }
  2586. return $tags;
  2587. }
  2588. private function getimage($id){
  2589. if(isset($this->js_image[$id])){
  2590. $return = $this->fuckhtml->parseJsString($this->js_image[$id]);
  2591. if(
  2592. $return != "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAUCAYAAACNiR0NAAABAUlEQVR4AWMYesChoYElLjkzPj4lY3d8csZjIL4MxPNjUzPcSTYsISFLAqj5NBD/h+LPQPwbiT87NCuLh2gDgRr2QzXuT0jNMoBYksARn5zuHJ+UcR0kB6RXE2VYXHJGOlTDZmzyIJcB5e+D1CSkZDgQNBAaZv+jU1JkcKpJygiGeZ0I76a/Byq8jU9NZFqaCNTA48SE33/iDcw8TIyBt0GKQTFN0Msp6f2EIyUpo57YSIlLSrMhIg0WCIBcCfXSdlzJBsheTHQ6jEnOUgEFOLaEDbMIlhZBOYrorAdJk+nroVnvPsSgdGdoOF7HZyhZ2XPoGQoqjbCpIbt0AiejIQMArVLI7k/DXFkAAAAASUVORK5CYII=" &&
  2593. $return != "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABgAAAAYCAYAAADgdz34AAAA6ElEQVR4Ae2UvQ7BYBSGW4uFxlVICLsYDA0D14NEunRxHSJ+BkYG9ibiHtgkuvpZWOod3uFESqpOF/ElT/q958h5OtQx/iexY/evY9ACJjBewUtkXHEPyBYUNQQuCETuggNrF2DHF3A4kfUMmLB+BoUYAg4nIX0TTNnbfCjg8HDBGuR4z4Ij+813giAC8rcrkXusjdQEpMpcYt5rCzrMaea7tqD9JLhpCyrMZeadpmApssPaUOszXQALGHz67De0/2gpMGPP014VFpizfgJ5zWXnAF8MryW1rj3x5l8LJANQF1lZQH5f8AAWpNcUs6HAEAAAAABJRU5ErkJggg=="
  2594. ){
  2595. if(
  2596. preg_match(
  2597. '/^\/\//',
  2598. $return
  2599. )
  2600. ){
  2601. return 'https:' . $return;
  2602. }
  2603. return $return;
  2604. }
  2605. return null;
  2606. }
  2607. }
  2608. private function parsecarousels(&$item_to_remove = false){
  2609. $carousels =
  2610. $this->fuckhtml
  2611. ->getElementsByClassName(
  2612. $this->findstyles(
  2613. [
  2614. "padding" => "16px",
  2615. "position" => "relative"
  2616. ],
  2617. self::is_class
  2618. )
  2619. );
  2620. $return = [];
  2621. for($i=0; $i<count($carousels); $i++){
  2622. if(!isset($carousels[$i]["outerHTML"])){
  2623. continue;
  2624. }
  2625. $this->fuckhtml->load($carousels[$i]);
  2626. if($item_to_remove !== false){
  2627. $item_to_remove =
  2628. str_replace(
  2629. $carousels[$i]["outerHTML"],
  2630. "",
  2631. $item_to_remove
  2632. );
  2633. }
  2634. $pcitems =
  2635. $this->fuckhtml
  2636. ->getElementsByClassName(
  2637. "pcitem",
  2638. "div"
  2639. );
  2640. foreach($pcitems as $pcitem){
  2641. $this->fuckhtml->load($pcitem);
  2642. $out = [
  2643. "url" => null,
  2644. "ref" => null,
  2645. "image" => null,
  2646. "thumb_width" => null,
  2647. "thumb_height" => null,
  2648. "image_width" => null,
  2649. "image_height" => null,
  2650. "title" => null,
  2651. "description" => null,
  2652. "subtext" => null,
  2653. "date" => null
  2654. ];
  2655. $url =
  2656. $this->unshiturl(
  2657. $this->fuckhtml
  2658. ->getElementsByTagName("a")
  2659. [0]
  2660. ["attributes"]
  2661. ["href"],
  2662. true
  2663. );
  2664. // set ref
  2665. $out["ref"] = $url["ref"];
  2666. // set url
  2667. $out["url"] = $url["url"];
  2668. // set sizes
  2669. $out["thumb_width"] = $url["thumb_width"];
  2670. $out["thumb_height"] = $url["thumb_height"];
  2671. $out["image_width"] = $url["image_width"];
  2672. $out["image_height"] = $url["image_height"];
  2673. // get image
  2674. $out["image"] =
  2675. $this->fuckhtml
  2676. ->getElementsByTagName(
  2677. "img"
  2678. );
  2679. if(count($out["image"]) !== 0){
  2680. // get title from image
  2681. if(isset($out["image"][0]["attributes"]["alt"])){
  2682. $out["title"] =
  2683. $this->titledots(
  2684. $this->fuckhtml
  2685. ->getTextContent(
  2686. $out["image"][0]["attributes"]["alt"]
  2687. )
  2688. );
  2689. }
  2690. // get image url
  2691. if(isset($out["image"][0]["attributes"]["id"])){
  2692. $out["image"] = $this->getimage($out["image"][0]["attributes"]["id"]);
  2693. }
  2694. elseif(isset($out["image"][0]["attributes"]["data-ll"])){
  2695. $out["image"] =
  2696. $this->fuckhtml
  2697. ->getTextContent(
  2698. $out["image"][0]["attributes"]["data-ll"]
  2699. );
  2700. }else{
  2701. // failed to get image information
  2702. $out["image"] = null;
  2703. }
  2704. if($out["image"] == 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABgAAAAYAgMAAACdGdVrAAAADFBMVEVMaXFChfRChfRChfT0tCPZAAAAA3RSTlMAgFJEkGxNAAAAL0lEQVR4AWPADxgdwBT3BTDF9AUiuhdC6WNK/v///y+UggrClSA07EWVglmEFwAA5eYSExeCwigAAAAASUVORK5CYII='){
  2705. // found arrow image base64, skip entry
  2706. continue;
  2707. }
  2708. }else{
  2709. // Could not find any image in node
  2710. $out["image"] = null;
  2711. }
  2712. // get title from spans
  2713. $title =
  2714. $this->fuckhtml
  2715. ->getElementsByClassName(
  2716. $this->findstyles(
  2717. [
  2718. "color" => "#1967d2"
  2719. ],
  2720. self::is_class
  2721. ),
  2722. "span"
  2723. );
  2724. if(count($title) !== 0){
  2725. $out["title"] =
  2726. $this->fuckhtml
  2727. ->getTextContent(
  2728. $title[0]
  2729. );
  2730. }
  2731. // get textnodes
  2732. $textnodes =
  2733. $this->fuckhtml
  2734. ->getElementsByClassName(
  2735. $this->findstyles(
  2736. [
  2737. "white-space" => "pre-line",
  2738. "word-wrap" => "break-word"
  2739. ],
  2740. self::is_class
  2741. )
  2742. );
  2743. $subtext = null;
  2744. if(count($textnodes) !== 0){
  2745. // get date
  2746. $date =
  2747. $this->fuckhtml
  2748. ->getTextContent(
  2749. $textnodes[count($textnodes) - 1],
  2750. true
  2751. );
  2752. if(str_replace("\n", " ", $date) == $title){
  2753. $date = null;
  2754. }else{
  2755. if(strpos($date, "\n") !== false){
  2756. $date = explode("\n", $date);
  2757. $date = $date[count($date) - 1];
  2758. }
  2759. elseif(strpos($date, "•") !== false){
  2760. $date = explode("•", $date);
  2761. $date = ltrim($date[count($date) - 1]);
  2762. }else{
  2763. $date = null;
  2764. }
  2765. }
  2766. if($date !== null){
  2767. $date = strtotime($date);
  2768. }
  2769. // get description
  2770. $description =
  2771. $this->fuckhtml
  2772. ->getTextContent(
  2773. $textnodes[0]
  2774. );
  2775. if($out["title"] === null){
  2776. if($date === null){
  2777. $out["title"] = $description;
  2778. $description = null;
  2779. }else{
  2780. $out["title"] = parse_url($out["url"], PHP_URL_HOST);
  2781. }
  2782. }
  2783. if(isset($textnodes[1])){
  2784. $out["subtext"] =
  2785. $this->fuckhtml
  2786. ->getTextContent(
  2787. $textnodes[1]
  2788. );
  2789. }
  2790. }else{
  2791. $date = null;
  2792. $description = null;
  2793. }
  2794. $out["date"] = $date;
  2795. $out["description"] = $this->titledots($description);
  2796. if($out["url"] === null){
  2797. $out["url"] = $out["title"];
  2798. }
  2799. if($out["title"] == $out["description"]){
  2800. $out["description"] = null;
  2801. }
  2802. $return[$i][] = $out;
  2803. }
  2804. }
  2805. return $return;
  2806. }
  2807. private function unshiturl($url, $return_size = false){
  2808. // get parameters from URL
  2809. $url =
  2810. $this->fuckhtml
  2811. ->getTextContent($url);
  2812. $newurl = parse_url($url, PHP_URL_QUERY);
  2813. if($newurl == ""){
  2814. // probably telephone number
  2815. return $url;
  2816. }
  2817. $url = $newurl;
  2818. unset($newurl);
  2819. parse_str($url, $query);
  2820. if(isset($query["imgurl"])){
  2821. $url = $query["imgurl"];
  2822. }
  2823. elseif(isset($query["q"])){
  2824. $url = $query["q"];
  2825. }
  2826. // rewrite URLs to remove extra tracking parameters
  2827. $domain = parse_url($url, PHP_URL_HOST);
  2828. if(
  2829. preg_match(
  2830. '/wikipedia.org$/',
  2831. $domain
  2832. )
  2833. ){
  2834. // rewrite wikipedia mobile URLs to desktop
  2835. $url =
  2836. $this->replacedomain(
  2837. $url,
  2838. preg_replace(
  2839. '/([a-z0-9]+)(\.m\.)/',
  2840. '$1.',
  2841. $domain
  2842. )
  2843. );
  2844. }
  2845. elseif(
  2846. preg_match(
  2847. '/imdb\.com$|youtube\.[^.]+$/',
  2848. $domain
  2849. )
  2850. ){
  2851. // rewrite imdb and youtube mobile URLs too
  2852. $url =
  2853. $this->replacedomain(
  2854. $url,
  2855. preg_replace(
  2856. '/^m\./',
  2857. "",
  2858. $domain
  2859. )
  2860. );
  2861. }
  2862. elseif(
  2863. preg_match(
  2864. '/play\.google\.[^.]+$/',
  2865. $domain
  2866. )
  2867. ){
  2868. // remove referrers from play.google.com
  2869. $oldquery = parse_url($url, PHP_URL_QUERY);
  2870. if($oldquery !== null){
  2871. parse_str($oldquery, $query);
  2872. if(isset($query["referrer"])){ unset($query["referrer"]); }
  2873. if(isset($query["hl"])){ unset($query["hl"]); }
  2874. if(isset($query["gl"])){ unset($query["gl"]); }
  2875. $query = http_build_query($query);
  2876. $url =
  2877. str_replace(
  2878. $oldquery,
  2879. $query,
  2880. $url
  2881. );
  2882. }
  2883. }
  2884. elseif(
  2885. preg_match(
  2886. '/twitter\.com$/',
  2887. $domain
  2888. )
  2889. ){
  2890. // remove more referrers from twitter.com
  2891. $oldquery = parse_url($url, PHP_URL_QUERY);
  2892. if($oldquery !== null){
  2893. parse_str($oldquery, $query);
  2894. if(isset($query["ref_src"])){ unset($query["ref_src"]); }
  2895. $query = http_build_query($query);
  2896. if($query != ""){
  2897. $query .= "?" . $query;
  2898. }
  2899. $url =
  2900. str_replace(
  2901. '?' . $oldquery,
  2902. $query,
  2903. $url
  2904. );
  2905. }
  2906. }
  2907. elseif(
  2908. preg_match(
  2909. '/maps\.google\.[^.]+/',
  2910. $domain
  2911. )
  2912. ){
  2913. if(stripos($url, "maps?") !== false){
  2914. //https://maps.google.com/maps?daddr=Johnny,+603+Rue+St+Georges,+Saint-J%C3%A9r%C3%B4me,+Quebec+J7Z+5B7
  2915. $query = parse_url($url, PHP_URL_QUERY);
  2916. if($query !== null){
  2917. parse_str($query, $query);
  2918. if(isset($query["daddr"])){
  2919. $url =
  2920. "https://maps.google.com/maps?daddr=" .
  2921. urlencode($query["daddr"]);
  2922. }
  2923. }
  2924. }
  2925. }
  2926. if($return_size){
  2927. return [
  2928. "url" => $url,
  2929. "ref" => isset($query["imgrefurl"]) ? $query["imgrefurl"] : null,
  2930. "thumb_width" => isset($query["tbnw"]) ? (int)$query["tbnw"] : null,
  2931. "thumb_height" => isset($query["tbnh"]) ? (int)$query["tbnh"] : null,
  2932. "image_width" => isset($query["w"]) ? (int)$query["w"] : null,
  2933. "image_height" => isset($query["h"]) ? (int)$query["h"] : null
  2934. ];
  2935. }
  2936. return $url;
  2937. }
  2938. private function replacedomain($url, $domain){
  2939. return
  2940. preg_replace(
  2941. '/(https?:\/\/)([^\/]+)/',
  2942. '$1' . $domain,
  2943. $url
  2944. );
  2945. }
  2946. private function titledots($title){
  2947. return rtrim($title, ". \t\n\r\0\x0B");
  2948. }
  2949. }