1
0

google.php 71 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639
  1. <?php
  2. // todo:
  3. // aliexpress tracking links
  4. // enhanced msx notice
  5. // detect "sorry" page
  6. class google{
  7. private const is_class = ".";
  8. private const is_id = "#";
  9. public function __construct(){
  10. include "lib/fuckhtml.php";
  11. $this->fuckhtml = new fuckhtml();
  12. include "lib/backend.php";
  13. $this->backend = new backend("google");
  14. }
  15. public function getfilters($page){
  16. $base = [
  17. "country" => [ // gl=<country>
  18. "display" => "Country",
  19. "option" => [
  20. "any" => "Instance's country",
  21. "af" => "Afghanistan",
  22. "al" => "Albania",
  23. "dz" => "Algeria",
  24. "as" => "American Samoa",
  25. "ad" => "Andorra",
  26. "ao" => "Angola",
  27. "ai" => "Anguilla",
  28. "aq" => "Antarctica",
  29. "ag" => "Antigua and Barbuda",
  30. "ar" => "Argentina",
  31. "am" => "Armenia",
  32. "aw" => "Aruba",
  33. "au" => "Australia",
  34. "at" => "Austria",
  35. "az" => "Azerbaijan",
  36. "bs" => "Bahamas",
  37. "bh" => "Bahrain",
  38. "bd" => "Bangladesh",
  39. "bb" => "Barbados",
  40. "by" => "Belarus",
  41. "be" => "Belgium",
  42. "bz" => "Belize",
  43. "bj" => "Benin",
  44. "bm" => "Bermuda",
  45. "bt" => "Bhutan",
  46. "bo" => "Bolivia",
  47. "ba" => "Bosnia and Herzegovina",
  48. "bw" => "Botswana",
  49. "bv" => "Bouvet Island",
  50. "br" => "Brazil",
  51. "io" => "British Indian Ocean Territory",
  52. "bn" => "Brunei Darussalam",
  53. "bg" => "Bulgaria",
  54. "bf" => "Burkina Faso",
  55. "bi" => "Burundi",
  56. "kh" => "Cambodia",
  57. "cm" => "Cameroon",
  58. "ca" => "Canada",
  59. "cv" => "Cape Verde",
  60. "ky" => "Cayman Islands",
  61. "cf" => "Central African Republic",
  62. "td" => "Chad",
  63. "cl" => "Chile",
  64. "cn" => "China",
  65. "cx" => "Christmas Island",
  66. "cc" => "Cocos (Keeling) Islands",
  67. "co" => "Colombia",
  68. "km" => "Comoros",
  69. "cg" => "Congo",
  70. "cd" => "Congo, the Democratic Republic",
  71. "ck" => "Cook Islands",
  72. "cr" => "Costa Rica",
  73. "ci" => "Cote D'ivoire",
  74. "hr" => "Croatia",
  75. "cu" => "Cuba",
  76. "cy" => "Cyprus",
  77. "cz" => "Czech Republic",
  78. "dk" => "Denmark",
  79. "dj" => "Djibouti",
  80. "dm" => "Dominica",
  81. "do" => "Dominican Republic",
  82. "ec" => "Ecuador",
  83. "eg" => "Egypt",
  84. "sv" => "El Salvador",
  85. "gq" => "Equatorial Guinea",
  86. "er" => "Eritrea",
  87. "ee" => "Estonia",
  88. "et" => "Ethiopia",
  89. "fk" => "Falkland Islands (Malvinas)",
  90. "fo" => "Faroe Islands",
  91. "fj" => "Fiji",
  92. "fi" => "Finland",
  93. "fr" => "France",
  94. "gf" => "French Guiana",
  95. "pf" => "French Polynesia",
  96. "tf" => "French Southern Territories",
  97. "ga" => "Gabon",
  98. "gm" => "Gambia",
  99. "ge" => "Georgia",
  100. "de" => "Germany",
  101. "gh" => "Ghana",
  102. "gi" => "Gibraltar",
  103. "gr" => "Greece",
  104. "gl" => "Greenland",
  105. "gd" => "Grenada",
  106. "gp" => "Guadeloupe",
  107. "gu" => "Guam",
  108. "gt" => "Guatemala",
  109. "gn" => "Guinea",
  110. "gw" => "Guinea-Bissau",
  111. "gy" => "Guyana",
  112. "ht" => "Haiti",
  113. "hm" => "Heard Island and Mcdonald Islands",
  114. "va" => "Holy See (Vatican City State)",
  115. "hn" => "Honduras",
  116. "hk" => "Hong Kong",
  117. "hu" => "Hungary",
  118. "is" => "Iceland",
  119. "in" => "India",
  120. "id" => "Indonesia",
  121. "ir" => "Iran, Islamic Republic",
  122. "iq" => "Iraq",
  123. "ie" => "Ireland",
  124. "il" => "Israel",
  125. "it" => "Italy",
  126. "jm" => "Jamaica",
  127. "jp" => "Japan",
  128. "jo" => "Jordan",
  129. "kz" => "Kazakhstan",
  130. "ke" => "Kenya",
  131. "ki" => "Kiribati",
  132. "kp" => "Korea, Democratic People's Republic",
  133. "kr" => "Korea, Republic",
  134. "kw" => "Kuwait",
  135. "kg" => "Kyrgyzstan",
  136. "la" => "Lao People's Democratic Republic",
  137. "lv" => "Latvia",
  138. "lb" => "Lebanon",
  139. "ls" => "Lesotho",
  140. "lr" => "Liberia",
  141. "ly" => "Libyan Arab Jamahiriya",
  142. "li" => "Liechtenstein",
  143. "lt" => "Lithuania",
  144. "lu" => "Luxembourg",
  145. "mo" => "Macao",
  146. "mk" => "Macedonia, the Former Yugosalv Republic",
  147. "mg" => "Madagascar",
  148. "mw" => "Malawi",
  149. "my" => "Malaysia",
  150. "mv" => "Maldives",
  151. "ml" => "Mali",
  152. "mt" => "Malta",
  153. "mh" => "Marshall Islands",
  154. "mq" => "Martinique",
  155. "mr" => "Mauritania",
  156. "mu" => "Mauritius",
  157. "yt" => "Mayotte",
  158. "mx" => "Mexico",
  159. "fm" => "Micronesia, Federated States",
  160. "md" => "Moldova, Republic",
  161. "mc" => "Monaco",
  162. "mn" => "Mongolia",
  163. "ms" => "Montserrat",
  164. "ma" => "Morocco",
  165. "mz" => "Mozambique",
  166. "mm" => "Myanmar",
  167. "na" => "Namibia",
  168. "nr" => "Nauru",
  169. "np" => "Nepal",
  170. "nl" => "Netherlands",
  171. "an" => "Netherlands Antilles",
  172. "nc" => "New Caledonia",
  173. "nz" => "New Zealand",
  174. "ni" => "Nicaragua",
  175. "ne" => "Niger",
  176. "ng" => "Nigeria",
  177. "nu" => "Niue",
  178. "nf" => "Norfolk Island",
  179. "mp" => "Northern Mariana Islands",
  180. "no" => "Norway",
  181. "om" => "Oman",
  182. "pk" => "Pakistan",
  183. "pw" => "Palau",
  184. "ps" => "Palestinian Territory, Occupied",
  185. "pa" => "Panama",
  186. "pg" => "Papua New Guinea",
  187. "py" => "Paraguay",
  188. "pe" => "Peru",
  189. "ph" => "Philippines",
  190. "pn" => "Pitcairn",
  191. "pl" => "Poland",
  192. "pt" => "Portugal",
  193. "pr" => "Puerto Rico",
  194. "qa" => "Qatar",
  195. "re" => "Reunion",
  196. "ro" => "Romania",
  197. "ru" => "Russian Federation",
  198. "rw" => "Rwanda",
  199. "sh" => "Saint Helena",
  200. "kn" => "Saint Kitts and Nevis",
  201. "lc" => "Saint Lucia",
  202. "pm" => "Saint Pierre and Miquelon",
  203. "vc" => "Saint Vincent and the Grenadines",
  204. "ws" => "Samoa",
  205. "sm" => "San Marino",
  206. "st" => "Sao Tome and Principe",
  207. "sa" => "Saudi Arabia",
  208. "sn" => "Senegal",
  209. "cs" => "Serbia and Montenegro",
  210. "sc" => "Seychelles",
  211. "sl" => "Sierra Leone",
  212. "sg" => "Singapore",
  213. "sk" => "Slovakia",
  214. "si" => "Slovenia",
  215. "sb" => "Solomon Islands",
  216. "so" => "Somalia",
  217. "za" => "South Africa",
  218. "gs" => "South Georgia and the South Sandwich Islands",
  219. "es" => "Spain",
  220. "lk" => "Sri Lanka",
  221. "sd" => "Sudan",
  222. "sr" => "Suriname",
  223. "sj" => "Svalbard and Jan Mayen",
  224. "sz" => "Swaziland",
  225. "se" => "Sweden",
  226. "ch" => "Switzerland",
  227. "sy" => "Syrian Arab Republic",
  228. "tw" => "Taiwan, Province of China",
  229. "tj" => "Tajikistan",
  230. "tz" => "Tanzania, United Republic",
  231. "th" => "Thailand",
  232. "tl" => "Timor-Leste",
  233. "tg" => "Togo",
  234. "tk" => "Tokelau",
  235. "to" => "Tonga",
  236. "tt" => "Trinidad and Tobago",
  237. "tn" => "Tunisia",
  238. "tr" => "Turkey",
  239. "tm" => "Turkmenistan",
  240. "tc" => "Turks and Caicos Islands",
  241. "tv" => "Tuvalu",
  242. "ug" => "Uganda",
  243. "ua" => "Ukraine",
  244. "ae" => "United Arab Emirates",
  245. "uk" => "United Kingdom",
  246. "us" => "United States",
  247. "um" => "United States Minor Outlying Islands",
  248. "uy" => "Uruguay",
  249. "uz" => "Uzbekistan",
  250. "vu" => "Vanuatu",
  251. "ve" => "Venezuela",
  252. "vn" => "Viet Nam",
  253. "vg" => "Virgin Islands, British",
  254. "vi" => "Virgin Islands, U.S.",
  255. "wf" => "Wallis and Futuna",
  256. "eh" => "Western Sahara",
  257. "ye" => "Yemen",
  258. "zm" => "Zambia",
  259. "zw" => "Zimbabwe"
  260. ]
  261. ],
  262. "nsfw" => [
  263. "display" => "NSFW",
  264. "option" => [
  265. "yes" => "Yes", // safe=active
  266. "no" => "No" // safe=off
  267. ]
  268. ],
  269. "lang" => [ // lr=<lang> (prefix lang with "lang_")
  270. "display" => "Language",
  271. "option" => [
  272. "any" => "Any language",
  273. "ar" => "Arabic",
  274. "bg" => "Bulgarian",
  275. "ca" => "Catalan",
  276. "cs" => "Czech",
  277. "da" => "Danish",
  278. "de" => "German",
  279. "el" => "Greek",
  280. "en" => "English",
  281. "es" => "Spanish",
  282. "et" => "Estonian",
  283. "fi" => "Finnish",
  284. "fr" => "French",
  285. "hr" => "Croatian",
  286. "hu" => "Hungarian",
  287. "id" => "Indonesian",
  288. "is" => "Icelandic",
  289. "it" => "Italian",
  290. "iw" => "Hebrew",
  291. "ja" => "Japanese",
  292. "ko" => "Korean",
  293. "lt" => "Lithuanian",
  294. "lv" => "Latvian",
  295. "nl" => "Dutch",
  296. "no" => "Norwegian",
  297. "pl" => "Polish",
  298. "pt" => "Portuguese",
  299. "ro" => "Romanian",
  300. "ru" => "Russian",
  301. "sk" => "Slovak",
  302. "sl" => "Slovenian",
  303. "sr" => "Serbian",
  304. "sv" => "Swedish",
  305. "tr" => "Turkish",
  306. "zh-CN" => "Chinese (Simplified)",
  307. "zh-TW" => "Chinese (Traditional)"
  308. ]
  309. ]
  310. ];
  311. switch($page){
  312. case "web":
  313. return array_merge(
  314. $base,
  315. [
  316. "newer" => [ // &sort=review-date:r:20090301:20090430
  317. "display" => "Newer than",
  318. "option" => "_DATE"
  319. ],
  320. "older" => [
  321. "display" => "Older than",
  322. "option" => "_DATE"
  323. ]
  324. ]
  325. );
  326. break;
  327. case "images":
  328. return array_merge(
  329. $base,
  330. [
  331. "time" => [ // tbs=qrd:<size>
  332. "display" => "Time posted",
  333. "option" => [
  334. "any" => "Any time",
  335. "d" => "Past 24 hours",
  336. "w" => "Past week",
  337. "m" => "Past month",
  338. "y" => "Past year"
  339. ]
  340. ],
  341. "size" => [
  342. "display" => "Size",
  343. "option" => [
  344. // tbs=isz:<size>
  345. "any" => "Any size",
  346. "l" => "Large",
  347. "m" => "Medium",
  348. "i" => "Icon",
  349. // from here
  350. // tbz:lt,islt:<size>
  351. "qsvga" => "Larger than 400x300",
  352. "vga" => "Larger than 640x480",
  353. "qsvga" => "Larger than 800x600",
  354. "xga" => "Larger than 1024x768",
  355. "2mp" => "Larger than 2MP",
  356. "4mp" => "Larger than 4MP",
  357. "6mp" => "Larger than 6MP",
  358. "8mp" => "Larger than 8MP",
  359. "10mp" => "Larger than 10MP",
  360. "12mp" => "Larger than 12MP",
  361. "15mp" => "Larger than 15MP",
  362. "20mp" => "Larger than 20MP",
  363. "40mp" => "Larger than 40MP",
  364. "70mp" => "Larger than 70MP"
  365. ]
  366. ],
  367. "ratio" => [ // tbs=iar:<size>
  368. "display" => "Aspect ratio",
  369. "option" => [
  370. "any" => "Any ratio",
  371. "t" => "Tall",
  372. "s" => "Square",
  373. "w" => "Wide",
  374. "xw" => "Panoramic"
  375. ]
  376. ],
  377. "color" => [ // tbs=ic:<color>
  378. "display" => "Color",
  379. "option" => [
  380. "any" => "Any color",
  381. "color" => "Full color",
  382. "gray" => "Black & white",
  383. "trans" => "Transparent",
  384. // from there, its ic:specific,isc:<color>
  385. "red" => "Red",
  386. "orange" => "Orange",
  387. "yellow" => "Yellow",
  388. "green" => "Green",
  389. "teal" => "Teal",
  390. "blue" => "Blue",
  391. "purple" => "Purple",
  392. "pink" => "Pink",
  393. "white" => "White",
  394. "gray" => "Gray",
  395. "black" => "Black",
  396. "brown" => "Brown"
  397. ]
  398. ],
  399. "type" => [ // tbs=itp:<type>
  400. "display" => "Type",
  401. "option" => [
  402. "any" => "Any type",
  403. "face" => "Faces",
  404. "clipart" => "Clip Art",
  405. "lineart" => "Line Drawing",
  406. "stock" => "Stock",
  407. "animated" => "Animated"
  408. ]
  409. ],
  410. "format" => [ // tbs=ift:<format>
  411. "display" => "Format",
  412. "option" => [
  413. "any" => "Any format",
  414. "jpg" => "JPG",
  415. "gif" => "GIF",
  416. "png" => "PNG",
  417. "bmp" => "BMP",
  418. "svg" => "SVG",
  419. "webp" => "WEBP",
  420. "ico" => "ICO",
  421. "craw" => "RAW"
  422. ]
  423. ],
  424. "rights" => [ // tbs=il:<rights>
  425. "display" => "Usage rights",
  426. "option" => [
  427. "any" => "Any license",
  428. "cl" => "Creative Commons licenses",
  429. "ol" => "Commercial & other licenses"
  430. ]
  431. ]
  432. ]
  433. );
  434. break;
  435. case "videos":
  436. return array_merge(
  437. $base,
  438. [
  439. "time" => [
  440. "display" => "Time posted",
  441. "option" => [ // tbs=qdr
  442. "any" => "Any time",
  443. "h" => "Past hour",
  444. "d" => "Past 24 hours",
  445. "w" => "Past week",
  446. "m" => "Past month",
  447. "y" => "Past year"
  448. ]
  449. ],
  450. "duration" => [
  451. "display" => "Duration",
  452. "option" => [
  453. "any" => "Any duration",
  454. "s" => "Short (0-4min)", // tbs=dur:s
  455. "m" => "Medium (4-20min)", // tbs=dur:m
  456. "l" => "Long (20+ min)" // tbs=dur:l
  457. ]
  458. ],
  459. "quality" => [
  460. "display" => "Quality",
  461. "option" => [
  462. "any" => "Any quality",
  463. "h" => "High quality" // tbs=hq:h
  464. ]
  465. ],
  466. "captions" => [
  467. "display" => "Captions",
  468. "option" => [
  469. "any" => "No preference",
  470. "yes" => "Closed captioned" // tbs=cc:1
  471. ]
  472. ]
  473. ]
  474. );
  475. break;
  476. case "news":
  477. return array_merge(
  478. $base,
  479. [
  480. "time" => [
  481. "display" => "Time posted",
  482. "option" => [ // tbs=qdr
  483. "any" => "Any time",
  484. "h" => "Past hour",
  485. "d" => "Past 24 hours",
  486. "w" => "Past week",
  487. "m" => "Past month",
  488. "y" => "Past year",
  489. "a" => "Archives" // tbs=ar:1
  490. ]
  491. ],
  492. "sort" => [
  493. "display" => "Sort",
  494. "option" => [
  495. "relevance" => "Relevance",
  496. "date" => "Date" // sbd:1
  497. ]
  498. ]
  499. ]
  500. );
  501. break;
  502. }
  503. }
  504. private function get($proxy, $url, $get = []){
  505. $headers = [
  506. "User-Agent: Mozilla/5.0 (Linux; U; Android 2.3.3; pt-pt; LG-P500h-parrot Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1 MMS/LG-Android-MMS-V1.0/1.2",
  507. "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
  508. "Accept-Language: en-US,en;q=0.5",
  509. "Accept-Encoding: gzip",
  510. "DNT: 1",
  511. "Connection: keep-alive",
  512. "Upgrade-Insecure-Requests: 1",
  513. "Sec-Fetch-Dest: document",
  514. "Sec-Fetch-Mode: navigate",
  515. "Sec-Fetch-Site: none",
  516. "Sec-Fetch-User: ?1"
  517. ];
  518. $curlproc = curl_init();
  519. if($get !== []){
  520. $get = http_build_query($get);
  521. $url .= "?" . $get;
  522. }
  523. curl_setopt($curlproc, CURLOPT_URL, $url);
  524. curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
  525. curl_setopt($curlproc, CURLOPT_HTTPHEADER, $headers);
  526. curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
  527. curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
  528. curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
  529. curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
  530. curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
  531. $this->backend->assign_proxy($curlproc, $proxy);
  532. $data = curl_exec($curlproc);
  533. if(curl_errno($curlproc)){
  534. throw new Exception(curl_error($curlproc));
  535. }
  536. curl_close($curlproc);
  537. echo $data;
  538. return $data;
  539. }
  540. public function web($get){
  541. if($get["npt"]){
  542. [$req, $ip] = $this->backend->get($get["npt"], "web");
  543. parse_str(
  544. parse_url($req, PHP_URL_QUERY),
  545. $search
  546. );
  547. if(isset($search["q"])){
  548. $search = $search["q"];
  549. }else{
  550. $search = "a"; // lol
  551. }
  552. try{
  553. $html =
  554. $this->get(
  555. $ip,
  556. "https://www.google.com" . $req,
  557. []
  558. );
  559. }catch(Exception $error){
  560. throw new Exception("Failed to get HTML");
  561. }
  562. }else{
  563. $search = $get["s"];
  564. $country = $get["country"];
  565. $nsfw = $get["nsfw"];
  566. $lang = $get["lang"];
  567. $older = $get["older"];
  568. $newer = $get["newer"];
  569. $ip = $this->backend->get_ip();
  570. $params = [
  571. "q" => $search,
  572. "hl" => "en",
  573. "num" => 20 // get 20 results
  574. ];
  575. // country
  576. if($country != "any"){
  577. $params["gl"] = $country;
  578. }
  579. // nsfw
  580. $params["safe"] = $nsfw == "yes" ? "off" : "active";
  581. // language
  582. if($lang != "any"){
  583. $params["lr"] = "lang_" . $lang;
  584. }
  585. // &sort=review-date:r:20090301:20090430
  586. $older = $older === false ? false : date("Ymd", $older);
  587. $newer = $newer === false ? false : date("Ymd", $newer);
  588. if(
  589. $older !== false &&
  590. $newer === false
  591. ){
  592. $newer = date("Ymd", time());
  593. }
  594. if(
  595. $older !== false ||
  596. $newer !== false
  597. ){
  598. $params["sort"] = "review-date:r:" . $older . ":" . $newer;
  599. }
  600. try{
  601. $html =
  602. $this->get(
  603. $ip,
  604. "https://www.google.com/search",
  605. $params
  606. );
  607. }catch(Exception $error){
  608. throw new Exception("Failed to get HTML");
  609. }
  610. }
  611. return $this->parsepage($html, "web", $search, $ip);
  612. }
  613. public function video($get){
  614. if($get["npt"]){
  615. [$req, $ip] = $this->backend->get($get["npt"], "videos");
  616. parse_str(
  617. parse_url($req, PHP_URL_QUERY),
  618. $search
  619. );
  620. if(isset($search["q"])){
  621. $search = $search["q"];
  622. }else{
  623. $search = "a"; // lol
  624. }
  625. try{
  626. $html =
  627. $this->get(
  628. $ip,
  629. "https://www.google.com" . $req,
  630. []
  631. );
  632. }catch(Exception $error){
  633. throw new Exception("Failed to get HTML");
  634. }
  635. }else{
  636. $search = $get["s"];
  637. $country = $get["country"];
  638. $nsfw = $get["nsfw"];
  639. $lang = $get["lang"];
  640. $time = $get["time"];
  641. $duration = $get["duration"];
  642. $quality = $get["quality"];
  643. $captions = $get["captions"];
  644. $ip = $this->backend->get_ip();
  645. $params = [
  646. "q" => $search,
  647. "tbm" => "vid",
  648. "hl" => "en",
  649. "num" => "20"
  650. ];
  651. // country
  652. if($country != "any"){
  653. $params["gl"] = $country;
  654. }
  655. // nsfw
  656. $params["safe"] = $nsfw == "yes" ? "off" : "active";
  657. // language
  658. if($lang != "any"){
  659. $params["lr"] = "lang_" . $lang;
  660. }
  661. $tbs = [];
  662. // time
  663. if($time != "any"){
  664. $tbs[] = "qdr:" . $time;
  665. }
  666. // duration
  667. if($duration != "any"){
  668. $tbs[] = "dur:" . $duration;
  669. }
  670. // quality
  671. if($quality != "any"){
  672. $tbs[] = "hq:" . $quality;
  673. }
  674. // captions
  675. if($captions != "any"){
  676. $tbs[] = "cc:" . $captions;
  677. }
  678. // append tbs
  679. if(count($tbs) !== 0){
  680. $params["tbs"] =
  681. implode(",", $tbs);
  682. }
  683. try{
  684. $html =
  685. $this->get(
  686. $ip,
  687. "https://www.google.com/search",
  688. $params
  689. );
  690. }catch(Exception $error){
  691. throw new Exception("Failed to get HTML");
  692. }
  693. }
  694. $json = $this->parsepage($html, "videos", $search, $ip);
  695. $out = [
  696. "status" => "ok",
  697. "npt" => $json["npt"],
  698. "video" => [],
  699. "author" => [],
  700. "livestream" => [],
  701. "playlist" => [],
  702. "reel" => []
  703. ];
  704. foreach($json["web"] as $item){
  705. $out["video"][] = [
  706. "title" => $item["title"],
  707. "description" => $item["description"],
  708. "author" => [
  709. "name" => null,
  710. "url" => null,
  711. "avatar" => null
  712. ],
  713. "date" => isset($item["table"]["Posted"]) ? strtotime($item["table"]["Posted"]) : null,
  714. "duration" => isset($item["table"]["Duration"]) ? $this->hms2int($item["table"]["Duration"]) : null,
  715. "views" => null,
  716. "thumb" =>
  717. $item["thumb"]["url"] === null ?
  718. [
  719. "url" => null,
  720. "ratio" => null
  721. ] :
  722. [
  723. "url" => $item["thumb"]["url"],
  724. "ratio" => "16:9"
  725. ],
  726. "url" => $item["url"]
  727. ];
  728. }
  729. return $out;
  730. }
  731. public function news($get){
  732. if($get["npt"]){
  733. [$req, $ip] = $this->backend->get($get["npt"], "news");
  734. parse_str(
  735. parse_url($req, PHP_URL_QUERY),
  736. $search
  737. );
  738. if(isset($search["q"])){
  739. $search = $search["q"];
  740. }else{
  741. $search = "a"; // lol
  742. }
  743. try{
  744. $html =
  745. $this->get(
  746. $ip,
  747. "https://www.google.com" . $req,
  748. []
  749. );
  750. }catch(Exception $error){
  751. throw new Exception("Failed to get HTML");
  752. }
  753. }else{
  754. $search = $get["s"];
  755. $country = $get["country"];
  756. $nsfw = $get["nsfw"];
  757. $lang = $get["lang"];
  758. $time = $get["time"];
  759. $sort = $get["sort"];
  760. $ip = $this->backend->get_ip();
  761. $params = [
  762. "q" => $search,
  763. "tbm" => "nws",
  764. "hl" => "en",
  765. "num" => "20"
  766. ];
  767. // country
  768. if($country != "any"){
  769. $params["gl"] = $country;
  770. }
  771. // nsfw
  772. $params["safe"] = $nsfw == "yes" ? "off" : "active";
  773. // language
  774. if($lang != "any"){
  775. $params["lr"] = "lang_" . $lang;
  776. }
  777. $tbs = [];
  778. // time
  779. if($time != "any"){
  780. if($time == "a"){
  781. $tbs[] = "ar:1";
  782. }else{
  783. $tbs[] = "qdr:" . $time;
  784. }
  785. }
  786. // relevance
  787. if($sort == "date"){
  788. $tbs[] = "sbd:1";
  789. }
  790. // append tbs
  791. if(count($tbs) !== 0){
  792. $params["tbs"] =
  793. implode(",", $tbs);
  794. }
  795. $html =
  796. $this->get(
  797. $ip,
  798. "https://www.google.com/search",
  799. $params
  800. );
  801. }
  802. $json = $this->parsepage($html, "news", $search, $ip);
  803. $out = [
  804. "status" => "ok",
  805. "npt" => $json["npt"],
  806. "news" => []
  807. ];
  808. foreach($json["web"] as $item){
  809. $description = array_key_first($item["table"]);
  810. if($description !== null){
  811. $date = $item["table"][$description];
  812. }else{
  813. $date = null;
  814. }
  815. $out["news"][] = [
  816. "title" => $item["title"],
  817. "author" => $item["author"],
  818. "description" => $description,
  819. "date" => strtotime($date),
  820. "thumb" =>
  821. $item["thumb"]["url"] === null ?
  822. [
  823. "url" => null,
  824. "ratio" => null
  825. ] :
  826. [
  827. "url" => $item["thumb"]["url"],
  828. "ratio" => "16:9"
  829. ],
  830. "url" => $item["url"]
  831. ];
  832. }
  833. return $out;
  834. }
  835. private function parsepage($html, $pagetype, $search, $ip){
  836. /*
  837. $handle = fopen("scraper/google.html", "r");
  838. $html = fread($handle, filesize("scraper/google.html"));
  839. fclose($handle);
  840. */
  841. $out = [
  842. "status" => "ok",
  843. "spelling" => [
  844. "type" => "no_correction",
  845. "using" => null,
  846. "correction" => null
  847. ],
  848. "npt" => null,
  849. "answer" => [],
  850. "web" => [],
  851. "image" => [],
  852. "video" => [],
  853. "news" => [],
  854. "related" => []
  855. ];
  856. if($this->detect_sorry($html)){
  857. throw new Exception("Google blocked this 4get instance. Please set up a proxy!");
  858. }
  859. $this->parsejavascript($html);
  860. //
  861. // parse accdef's
  862. //
  863. $has_appended_accdef = false;
  864. preg_match_all(
  865. '/window\.jsl\.dh\(\'(accdef_[0-9]+)\',\'(.*)\'\);/',
  866. $html,
  867. $accdefs_regex
  868. );
  869. $accdefs = [];
  870. for($i=0; $i<count($accdefs_regex[0]); $i++){
  871. // decode UTF-16 string
  872. $answer =
  873. $this->fuckhtml
  874. ->parseJsString(
  875. $accdefs_regex[2][$i]
  876. );
  877. $this->fuckhtml->load($answer);
  878. // get description
  879. $description =
  880. $this->fuckhtml
  881. ->getElementsByClassName(
  882. $this->findstyles(
  883. [
  884. "padding" => "12px 16px 12px",
  885. ],
  886. self::is_class
  887. ),
  888. "div"
  889. )[1];
  890. // get date (rare)
  891. $date =
  892. $this->fuckhtml
  893. ->getElementsByTagName("sub");
  894. if(count($date) !== 0){
  895. $description =
  896. str_replace(
  897. $date[0]["outerHTML"],
  898. "",
  899. $description["innerHTML"]
  900. );
  901. $date =
  902. strtotime(
  903. $this->fuckhtml
  904. ->getTextContent(
  905. $date[0]
  906. )
  907. );
  908. }else{
  909. $date = null;
  910. }
  911. // get information table
  912. $table = [];
  913. $tbody =
  914. $this->fuckhtml
  915. ->getElementsByTagName("tbody");
  916. if(count($tbody) !== 0){
  917. $this->fuckhtml->load($tbody[0]);
  918. $trs =
  919. $this->fuckhtml
  920. ->getElementsByTagName("tr");
  921. foreach($trs as $tr){
  922. $this->fuckhtml->load($tr);
  923. $tds =
  924. $this->fuckhtml
  925. ->getElementsByTagName("td");
  926. if(count($tds) === 2){
  927. $table[
  928. $this->fuckhtml
  929. ->getTextContent(
  930. $tds[0]
  931. )
  932. ] =
  933. $this->fuckhtml
  934. ->getTextContent(
  935. $tds[1]
  936. );
  937. }
  938. }
  939. // load back what we had
  940. $this->fuckhtml->load($answer);
  941. }
  942. // get title & link
  943. $a =
  944. $this->fuckhtml
  945. ->getElementsByTagName("a")[0];
  946. $this->fuckhtml->load($a);
  947. $title =
  948. $this->fuckhtml
  949. ->getElementsByTagName("span");
  950. if(count($title) === 0){
  951. continue;
  952. }
  953. $accdefs[] = [
  954. "title" =>
  955. $this->titledots(
  956. $this->fuckhtml
  957. ->getTextContent(
  958. $title[0]
  959. )
  960. ),
  961. "description" =>
  962. $this->fuckhtml
  963. ->getTextContent(
  964. $description
  965. ),
  966. "url" =>
  967. $this->unshiturl(
  968. $a["attributes"]["href"]
  969. ),
  970. "date" => $date,
  971. "type" => "web",
  972. "thumb" => [
  973. "url" => null,
  974. "ratio" => null
  975. ],
  976. "sublink" => [],
  977. "table" => $table
  978. ];
  979. }
  980. $this->fuckhtml->load($html);
  981. $containers =
  982. $this->fuckhtml
  983. ->getElementsByClassName(
  984. $this->findstyles(
  985. [
  986. "background-color" => "#fff",
  987. "margin-bottom" => "10px",
  988. "-webkit-box-shadow" => "0 1px 6px rgba(32,33,36,0.28)",
  989. "border-radius" => "8px"
  990. ],
  991. self::is_class
  992. ),
  993. "div"
  994. );
  995. foreach($containers as $container){
  996. $this->fuckhtml->load($container);
  997. // detect spelling
  998. $spelling =
  999. $this->fuckhtml
  1000. ->getElementsByClassName(
  1001. $this->findstyles(
  1002. [
  1003. "font-size" => "20px",
  1004. "line-height" => "26px",
  1005. "padding-top" => "2px",
  1006. "margin-bottom" => "1px"
  1007. ],
  1008. self::is_class
  1009. ),
  1010. "div"
  1011. );
  1012. if(count($spelling) !== 0){
  1013. $a =
  1014. $this->fuckhtml
  1015. ->getElementsByTagName("a");
  1016. if(count($a) !== 0){
  1017. $scripts =
  1018. $this->fuckhtml
  1019. ->getElementsByTagName("script");
  1020. foreach($scripts as $script){
  1021. $container["innerHTML"] =
  1022. str_replace(
  1023. $script["outerHTML"],
  1024. "",
  1025. $container["innerHTML"]
  1026. );
  1027. }
  1028. $container["innerHTML"] =
  1029. $this->fuckhtml
  1030. ->getTextContent(
  1031. str_replace(
  1032. $a[0]["outerHTML"],
  1033. "",
  1034. $container["innerHTML"]
  1035. )
  1036. );
  1037. if(
  1038. preg_match(
  1039. '/^did you mean/i',
  1040. $container["innerHTML"]
  1041. )
  1042. ){
  1043. $out["spelling"] = [
  1044. "type" => "not_many",
  1045. "using" => $search,
  1046. "correction" =>
  1047. $this->fuckhtml
  1048. ->getTextContent(
  1049. $a[0]
  1050. )
  1051. ];
  1052. }
  1053. elseif(
  1054. preg_match(
  1055. '/^showing results for/i',
  1056. $container["innerHTML"]
  1057. )
  1058. ){
  1059. $out["spelling"] = [
  1060. "type" => "including",
  1061. "using" =>
  1062. $this->fuckhtml
  1063. ->getTextContent(
  1064. $a[0]
  1065. ),
  1066. "correction" => $search
  1067. ];
  1068. }
  1069. }
  1070. continue;
  1071. }
  1072. $title =
  1073. $this->fuckhtml
  1074. ->getElementsByClassName(
  1075. $this->findstyles(
  1076. [
  1077. "color" => "#1967d2",
  1078. "font-size" => "20px",
  1079. "line-height" => "26px"
  1080. ],
  1081. self::is_class
  1082. ),
  1083. "div"
  1084. );
  1085. if(count($title) !== 0){
  1086. //
  1087. // Container is a web link
  1088. //
  1089. $web = [
  1090. "title" =>
  1091. $this->titledots(
  1092. $this->fuckhtml
  1093. ->getTextContent(
  1094. $title[0]
  1095. )
  1096. ),
  1097. "description" => null,
  1098. "url" => null,
  1099. "date" => null,
  1100. "type" => "web",
  1101. "thumb" => [
  1102. "url" => null,
  1103. "ratio" => null
  1104. ],
  1105. "sublink" => [],
  1106. "table" => []
  1107. ];
  1108. // get link
  1109. $web["url"] =
  1110. $this->unshiturl(
  1111. $this->fuckhtml
  1112. ->getElementsByTagName("a")
  1113. [0]
  1114. ["attributes"]
  1115. ["href"]
  1116. );
  1117. //
  1118. // check if link contains a carousel
  1119. //
  1120. $carousels = $this->parsecarousels();
  1121. if(count($carousels) !== 0){
  1122. $first = true;
  1123. foreach($carousels as $carousel_cat){
  1124. foreach($carousel_cat as $carousel){
  1125. if($first === true){
  1126. $first = false;
  1127. }elseif($carousel["image"] !== null){
  1128. $out["image"][] = [
  1129. "title" => $carousel["title"],
  1130. "source" => [
  1131. [
  1132. "url" => $carousel["image"],
  1133. "width" => null,
  1134. "height" => null
  1135. ]
  1136. ],
  1137. "url" => $carousel["url"]
  1138. ];
  1139. }
  1140. $web["sublink"][] = [
  1141. "title" => $carousel["title"],
  1142. "date" => $carousel["date"],
  1143. "description" => $carousel["description"],
  1144. "url" => $carousel["url"]
  1145. ];
  1146. }
  1147. }
  1148. if($carousels[0][0]["image"] !== null){
  1149. $web["thumb"] = [
  1150. "url" => $carousels[0][0]["image"],
  1151. "ratio" => "16:9"
  1152. ];
  1153. }
  1154. $out["web"][] = $web;
  1155. continue;
  1156. }
  1157. //
  1158. // no carousel entries, parse as normal link
  1159. //
  1160. $this->fuckhtml->load($container);
  1161. // parse URL
  1162. $web["url"] =
  1163. $this->unshiturl(
  1164. $this->fuckhtml
  1165. ->getElementsByTagName("a")
  1166. [0]
  1167. ["attributes"]
  1168. ["href"]
  1169. );
  1170. $container = $container["innerHTML"];
  1171. $line_detect =
  1172. $this->fuckhtml
  1173. ->getElementsByClassName(
  1174. $this->findstyles(
  1175. [
  1176. "height" => "1px",
  1177. "background-color" => "#dadce0",
  1178. "margin" => "0 16px"
  1179. ],
  1180. self::is_class
  1181. ),
  1182. "div"
  1183. );
  1184. if(count($line_detect) !== 0){
  1185. // we found a line, this means we're dealing with a
  1186. // "featured snippet"
  1187. $featured = true;
  1188. $description_container =
  1189. $this->fuckhtml
  1190. ->getElementsByClassName(
  1191. $this->findstyles(
  1192. [
  1193. "white-space" => "pre-line",
  1194. "word-wrap" => "break-word"
  1195. ],
  1196. self::is_class
  1197. ),
  1198. "div"
  1199. )[1];
  1200. // get date node for it
  1201. $date =
  1202. $this->fuckhtml
  1203. ->getElementsByTagName("sub");
  1204. if(count($date) !== 0){
  1205. $web["date"] =
  1206. strtotime(
  1207. $this->fuckhtml
  1208. ->getTextContent(
  1209. $date[0]
  1210. )
  1211. );
  1212. }
  1213. }else{
  1214. // we're dealing with a normal link
  1215. $featured = false;
  1216. $description_container =
  1217. $this->fuckhtml
  1218. ->getElementsByClassName(
  1219. $this->findstyles(
  1220. [
  1221. "padding" => "12px 16px 12px"
  1222. ],
  1223. self::is_class
  1224. ),
  1225. "div"
  1226. )[1];
  1227. }
  1228. //
  1229. // Get author if we're parsing news
  1230. //
  1231. if($pagetype == "news"){
  1232. $author =
  1233. $this->fuckhtml
  1234. ->getElementsByClassName(
  1235. $this->findstyles(
  1236. [
  1237. "position" => "absolute",
  1238. "width" => "100%",
  1239. "top" => "0",
  1240. "left" => "0",
  1241. "padding-top" => "1px",
  1242. "margin-bottom" => "-1px"
  1243. ],
  1244. self::is_class
  1245. ),
  1246. "div"
  1247. );
  1248. if(count($author) !== 0){
  1249. $web["author"] =
  1250. $this->fuckhtml
  1251. ->getTextContent(
  1252. $author[0]
  1253. );
  1254. }else{
  1255. $web["author"] = null;
  1256. }
  1257. }
  1258. $description =
  1259. $description_container["innerHTML"];
  1260. $this->fuckhtml->load($description);
  1261. //
  1262. // get thumbnail before we call loadhtml again
  1263. //
  1264. $img =
  1265. $this->fuckhtml
  1266. ->getElementsByTagName("img");
  1267. if(count($img) !== 0){
  1268. $skip = true;
  1269. if(
  1270. isset($img[0]["attributes"]["alt"]) &&
  1271. stripos($img[0]["attributes"]["alt"], "Video for") !== false
  1272. ){
  1273. // is a video thumbnail
  1274. $web["thumb"]["ratio"] = "16:9";
  1275. }else{
  1276. // is a google thumbnail
  1277. $web["thumb"]["ratio"] = "1:1";
  1278. }
  1279. $web["thumb"]["url"] =
  1280. $this->getimage(
  1281. $img[0]["attributes"]["id"]
  1282. );
  1283. }else{
  1284. $skip = false;
  1285. }
  1286. //
  1287. // get sublinks
  1288. //
  1289. $links =
  1290. $this->fuckhtml
  1291. ->getElementsByTagName("a");
  1292. foreach($links as $link){
  1293. if($skip === true){
  1294. $skip = false;
  1295. continue;
  1296. }
  1297. $description =
  1298. str_replace(
  1299. $link["outerHTML"],
  1300. "",
  1301. $description
  1302. );
  1303. $sublink = [
  1304. "title" => null,
  1305. "description" => null,
  1306. "url" => null,
  1307. "date" => null
  1308. ];
  1309. $sublink["title"] =
  1310. $this->titledots(
  1311. $this->fuckhtml
  1312. ->getTextContent(
  1313. $link
  1314. )
  1315. );
  1316. $sublink["url"] =
  1317. $this->unshiturl(
  1318. $link
  1319. ["attributes"]
  1320. ["href"]
  1321. );
  1322. if(parse_url($sublink["url"], PHP_URL_HOST) !== null){
  1323. $web["sublink"][] = $sublink;
  1324. }
  1325. }
  1326. //
  1327. // Parse spans in description
  1328. //
  1329. $this->fuckhtml->load($description);
  1330. if($featured === false){
  1331. $levels =
  1332. $this->fuckhtml
  1333. ->getElementsByClassName(
  1334. $this->findstyles(
  1335. [
  1336. "padding-bottom" => "8px"
  1337. ],
  1338. self::is_class
  1339. ),
  1340. "div"
  1341. );
  1342. // oh my god yes, fucking great, sometimes there are NO levels
  1343. // hahahahahhahahahahahahahahahhahaa
  1344. if(count($levels) === 0){
  1345. $levels = [$description];
  1346. }
  1347. foreach($levels as $level){
  1348. $this->fuckhtml->load($level);
  1349. $spans =
  1350. $this->fuckhtml
  1351. ->getElementsByTagName(
  1352. "span"
  1353. );
  1354. $is_rating = -1;
  1355. foreach($spans as $span){
  1356. $innertext =
  1357. trim(
  1358. $this->fuckhtml
  1359. ->getTextContent(
  1360. $span
  1361. ),
  1362. " ·."
  1363. );
  1364. if($innertext == ""){ continue; }
  1365. if(
  1366. strtolower($innertext)
  1367. == "rating"
  1368. ){
  1369. $is_rating = 0;
  1370. // clean up before we go
  1371. $description =
  1372. str_replace(
  1373. $span["outerHTML"],
  1374. "",
  1375. $description
  1376. );
  1377. continue;
  1378. }
  1379. //
  1380. // Parse rating object
  1381. //
  1382. if($is_rating >= 0){
  1383. // clean up description
  1384. $description =
  1385. str_replace(
  1386. $span["outerHTML"],
  1387. "",
  1388. $description
  1389. );
  1390. if($span["level"] !== 1){ continue; }
  1391. $is_rating++;
  1392. // 10/10 (123)
  1393. if($is_rating === 1){
  1394. $innertext = explode(" ", $innertext, 2);
  1395. $web["table"]["Rating"] = $innertext[0];
  1396. if(count($innertext) === 2){
  1397. $web["table"]["Hits"] =
  1398. trim(
  1399. str_replace(
  1400. [
  1401. "(",
  1402. ")"
  1403. ],
  1404. "",
  1405. $innertext[1]
  1406. )
  1407. );
  1408. if($web["table"]["Hits"] == ""){
  1409. unset($web["table"]["Hits"]);
  1410. }
  1411. }
  1412. continue;
  1413. }
  1414. // US$4.99
  1415. // MYR 50.00
  1416. // $38.34
  1417. // JP¥6,480
  1418. // Reviewed by your mom
  1419. if($is_rating === 2){
  1420. if(
  1421. preg_match(
  1422. '/^Review by (.+)/',
  1423. $innertext,
  1424. $match
  1425. )
  1426. ){
  1427. $web["table"]["Author"] = $match[1];
  1428. continue;
  1429. }
  1430. $web["table"]["Price"] = $innertext;
  1431. continue;
  1432. }
  1433. // Android / In stock
  1434. if($is_rating === 3){
  1435. $web["table"]["Support"] = $innertext;
  1436. continue;
  1437. }
  1438. // ignore the rest
  1439. continue;
  1440. }
  1441. //
  1442. // Parse standalone text
  1443. //
  1444. // If we reach this point:
  1445. // 1. Ratings have been parsed
  1446. // 2. We're parsing a WEB link, not some shitty piece of shit
  1447. // check for date
  1448. // if span has no text before it, assume it's a date
  1449. $desc_split =
  1450. explode(
  1451. $span["outerHTML"],
  1452. $description,
  1453. 2
  1454. );
  1455. if(
  1456. $this->fuckhtml
  1457. ->getTextContent(
  1458. $desc_split[0]
  1459. ) == ""
  1460. ){
  1461. // has no text before
  1462. $date = strtotime($innertext);
  1463. if($date){
  1464. $web["date"] = $date;
  1465. }
  1466. // cleanup
  1467. $description =
  1468. str_replace(
  1469. $span["outerHTML"],
  1470. "",
  1471. $description
  1472. );
  1473. continue;
  1474. }
  1475. // Ready to parse table
  1476. if(count($desc_split) === 2){
  1477. $this->fuckhtml->load($desc_split[1]);
  1478. $web["table"][
  1479. $this->fuckhtml
  1480. ->getTextContent(
  1481. trim($desc_split[0], ": ")
  1482. )
  1483. ] = $innertext;
  1484. // cleanup
  1485. $description =
  1486. str_replace(
  1487. $desc_split[0] . $span["outerHTML"],
  1488. "",
  1489. $description
  1490. );
  1491. }
  1492. }
  1493. }
  1494. }
  1495. $web["description"] =
  1496. trim(
  1497. $this->fuckhtml
  1498. ->getTextContent(
  1499. $description
  1500. ),
  1501. " ·."
  1502. );
  1503. if($web["description"] == ""){
  1504. $web["description"] = null;
  1505. }
  1506. $out["web"][] = $web;
  1507. continue;
  1508. }
  1509. //
  1510. // Detect wikipedia shit
  1511. //
  1512. $wiki_title =
  1513. $this->fuckhtml
  1514. ->getElementsByTagName("h3");
  1515. if(count($wiki_title) !== 0){
  1516. $description_after = [];
  1517. $description = [];
  1518. $table = [];
  1519. $sublink = [];
  1520. $as =
  1521. $this->fuckhtml
  1522. ->getElementsByTagName("a");
  1523. foreach($as as $a){
  1524. if(
  1525. isset($a["attributes"]["href"]) &&
  1526. parse_url($a["attributes"]["href"], PHP_URL_HOST) == "maps.google.com"
  1527. ){
  1528. // detected maps embed, ignore
  1529. continue 2;
  1530. }
  1531. }
  1532. // get carousels and remove them from container for image grepper
  1533. $carousels = $this->parsecarousels($container["innerHTML"]);
  1534. $this->fuckhtml->load($container);
  1535. // add images to image tab, if applicable
  1536. for($i=0; $i<count($carousels); $i++){
  1537. foreach($carousels[$i] as $item){
  1538. if(
  1539. $item["url"] !== null &&
  1540. $item["ref"] !== null &&
  1541. $item["image"] !== null &&
  1542. $item["title"] !== null
  1543. ){
  1544. $out["image"][] = [
  1545. "title" => $item["title"],
  1546. "source" => [
  1547. [
  1548. "url" => $item["url"],
  1549. "width" => $item["image_width"],
  1550. "height" => $item["image_height"]
  1551. ],
  1552. [
  1553. "url" => $item["image"],
  1554. "width" => $item["thumb_width"],
  1555. "height" => $item["thumb_height"]
  1556. ]
  1557. ],
  1558. "url" => $item["ref"]
  1559. ];
  1560. unset($carousels[$i]);
  1561. }
  1562. }
  1563. }
  1564. $carousels = array_values($carousels);
  1565. // interpret remaining carousels as title + carousel
  1566. $titles =
  1567. $this->fuckhtml
  1568. ->getElementsByClassName(
  1569. $this->findstyles(
  1570. [
  1571. "font-weight" => "700",
  1572. "letter-spacing" => "0.75px",
  1573. "text-transform" => "uppercase"
  1574. ],
  1575. self::is_class
  1576. )
  1577. );
  1578. for($i=0; $i<count($titles); $i++){
  1579. if(!isset($carousels[$i])){
  1580. break;
  1581. }
  1582. $description_after[] = [
  1583. "type" => "title",
  1584. "value" =>
  1585. $this->fuckhtml
  1586. ->getTextContent(
  1587. $titles[$i]
  1588. )
  1589. ];
  1590. foreach($carousels[$i] as $carousel){
  1591. $description_after[] = [
  1592. "type" => "link",
  1593. "url" => "web?s=" . urlencode($carousel["description"]) . "&scraper=google",
  1594. "value" => $carousel["description"]
  1595. ];
  1596. if($carousel["subtext"] !== null){
  1597. $description_after[] = [
  1598. "type" => "quote",
  1599. "value" => $carousel["subtext"]
  1600. ];
  1601. }
  1602. $description_after[] = [
  1603. "type" => "image",
  1604. "url" => $carousel["image"]
  1605. ];
  1606. }
  1607. }
  1608. $categories =
  1609. $this->fuckhtml
  1610. ->getElementsByClassName(
  1611. $this->findstyles(
  1612. [
  1613. "padding" => "12px 16px 12px"
  1614. ],
  1615. self::is_class
  1616. )
  1617. );
  1618. $image =
  1619. $this->fuckhtml
  1620. ->getElementsByTagName("img");
  1621. if(count($image) !== 0){
  1622. $image = $this->getimage($image[0]["attributes"]["id"]);
  1623. }else{
  1624. $image = null;
  1625. }
  1626. $url = null;
  1627. for($i=0; $i<count($categories); $i++){
  1628. $this->fuckhtml->load($categories[$i]);
  1629. if($i === 0){
  1630. // first node. this should be the header with the small
  1631. // information snippet
  1632. $url =
  1633. $this->fuckhtml
  1634. ->getElementsByTagName("a");
  1635. if(count($url) !== 0){
  1636. $url =
  1637. $this->unshiturl(
  1638. $url[0]["attributes"]["href"]
  1639. );
  1640. if(parse_url($url, PHP_URL_HOST) == "encrypted-tbn0.gstatic.com"){
  1641. $image = $url;
  1642. $url = null;
  1643. }
  1644. }else{
  1645. $url = null;
  1646. }
  1647. $categories[$i]["innerHTML"] =
  1648. str_replace(
  1649. $wiki_title[0]["outerHTML"],
  1650. "",
  1651. $categories[$i]["innerHTML"]
  1652. );
  1653. $subtext =
  1654. $this->fuckhtml
  1655. ->getTextContent(
  1656. $categories[$i]["innerHTML"]
  1657. );
  1658. if(strlen($subtext) !== 0){
  1659. $description[] = [
  1660. "type" => "quote",
  1661. "value" =>
  1662. $this->fuckhtml
  1663. ->getTextContent(
  1664. $categories[$i]["innerHTML"]
  1665. )
  1666. ];
  1667. }
  1668. // detect audio file
  1669. $audio =
  1670. $this->fuckhtml
  1671. ->getElementsByTagName("audio");
  1672. if(count($audio) !== 0){
  1673. $description[] = [
  1674. "type" => "audio",
  1675. "url" =>
  1676. $this->fuckhtml
  1677. ->getTextContent(
  1678. $audio[0]["attributes"]["src"]
  1679. )
  1680. ];
  1681. }
  1682. }else{
  1683. // check for separator elements IN THERE
  1684. $separators =
  1685. $this->fuckhtml
  1686. ->getElementsByClassName(
  1687. $this->findstyles(
  1688. [
  1689. "white-space" => "pre-line",
  1690. "word-wrap" => "break-word"
  1691. ],
  1692. self::is_class
  1693. ),
  1694. "div"
  1695. );
  1696. // detect container type
  1697. foreach($separators as $separator){
  1698. $this->fuckhtml->load($separator);
  1699. // ignore wrong levels
  1700. if($separator["level"] !== 2){
  1701. continue;
  1702. }
  1703. //
  1704. // Detect word definition
  1705. //
  1706. $wordwraps =
  1707. $this->fuckhtml
  1708. ->getElementsByClassName(
  1709. $this->findstyles(
  1710. [
  1711. "padding-bottom" => "12px"
  1712. ],
  1713. self::is_class
  1714. ),
  1715. "div"
  1716. );
  1717. if(count($wordwraps) !== 0){
  1718. foreach($wordwraps as $word){
  1719. $this->fuckhtml->load($word);
  1720. // detect title
  1721. $span =
  1722. $this->fuckhtml
  1723. ->getElementsByTagName(
  1724. "span"
  1725. );
  1726. if(
  1727. count($span) === 1 &&
  1728. $this->fuckhtml
  1729. ->getTextContent(
  1730. str_replace(
  1731. $span[0]["outerHTML"],
  1732. "",
  1733. $word["innerHTML"]
  1734. )
  1735. ) == ""
  1736. ){
  1737. $description[] = [
  1738. "type" => "title",
  1739. "value" =>
  1740. $this->fuckhtml
  1741. ->getTextContent(
  1742. $span[0]
  1743. )
  1744. ];
  1745. continue;
  1746. }
  1747. // detect list element
  1748. $lists =
  1749. $this->fuckhtml
  1750. ->getElementsByTagName("ol");
  1751. if(count($lists) !== 0){
  1752. foreach($lists as $list){
  1753. $this->fuckhtml->load($list);
  1754. $items =
  1755. $this->fuckhtml
  1756. ->getElementsByTagName("li");
  1757. $w = 0;
  1758. foreach($items as $item){
  1759. $w++;
  1760. $this->fuckhtml->load($item);
  1761. // get subnodes
  1762. $subnodes =
  1763. $this->fuckhtml
  1764. ->getElementsByClassName(
  1765. $this->findstyles(
  1766. [
  1767. "white-space" => "pre-line",
  1768. "word-wrap" => "break-word"
  1769. ],
  1770. self::is_class
  1771. ),
  1772. "div"
  1773. );
  1774. foreach($subnodes as $subnode){
  1775. $this->fuckhtml->load($subnode);
  1776. $spans =
  1777. $this->fuckhtml
  1778. ->getElementsByTagName("span");
  1779. if(count($spans) !== 0){
  1780. // append quote
  1781. $description[] = [
  1782. "type" => "quote",
  1783. "value" =>
  1784. $this->fuckhtml
  1785. ->getTextContent(
  1786. $subnode
  1787. )
  1788. ];
  1789. }else{
  1790. // append text
  1791. $description[] = [
  1792. "type" => "text",
  1793. "value" =>
  1794. $w . ". " .
  1795. $this->fuckhtml
  1796. ->getTextContent(
  1797. $subnode
  1798. )
  1799. ];
  1800. }
  1801. }
  1802. }
  1803. }
  1804. }else{
  1805. // parse without list
  1806. // get subnodes
  1807. $subnodes =
  1808. $this->fuckhtml
  1809. ->getElementsByClassName(
  1810. $this->findstyles(
  1811. [
  1812. "white-space" => "pre-line",
  1813. "word-wrap" => "break-word"
  1814. ],
  1815. self::is_class
  1816. ),
  1817. "div"
  1818. );
  1819. foreach($subnodes as $subnode){
  1820. $this->fuckhtml->load($subnode);
  1821. $spans =
  1822. $this->fuckhtml
  1823. ->getElementsByTagName("span");
  1824. if(count($spans) !== 0){
  1825. // append quote
  1826. $description[] = [
  1827. "type" => "quote",
  1828. "value" =>
  1829. $this->fuckhtml
  1830. ->getTextContent(
  1831. $subnode
  1832. )
  1833. ];
  1834. }else{
  1835. // append text
  1836. $description[] = [
  1837. "type" => "text",
  1838. "value" =>
  1839. $this->fuckhtml
  1840. ->getTextContent(
  1841. $subnode
  1842. )
  1843. ];
  1844. }
  1845. }
  1846. }
  1847. }
  1848. }else{
  1849. //
  1850. // Parse table
  1851. //
  1852. $spans =
  1853. $this->fuckhtml
  1854. ->getElementsByTagName("span");
  1855. foreach($spans as $span){
  1856. if(!isset($span["attributes"]["class"])){
  1857. // found table
  1858. $row =
  1859. explode(
  1860. ":",
  1861. $this->fuckhtml
  1862. ->getTextContent(
  1863. $separator
  1864. ),
  1865. 2
  1866. );
  1867. if(count($row) === 2){
  1868. $table[rtrim($row[0])] =
  1869. ltrim($row[1]);
  1870. }
  1871. continue 2;
  1872. }
  1873. }
  1874. //
  1875. // Parse normal description
  1876. //
  1877. $links_rem =
  1878. $this->fuckhtml
  1879. ->getElementsByTagName("a");
  1880. foreach($links_rem as $rem){
  1881. $separator["innerHTML"] =
  1882. str_replace(
  1883. $rem["outerHTML"],
  1884. "",
  1885. $separator["innerHTML"]
  1886. );
  1887. }
  1888. $description[] = [
  1889. "type" => "text",
  1890. "value" =>
  1891. rtrim(
  1892. $this->fuckhtml
  1893. ->getTextContent(
  1894. $separator
  1895. ),
  1896. " .,"
  1897. )
  1898. ];
  1899. }
  1900. }
  1901. }
  1902. // detect huge buttons
  1903. $buttons =
  1904. $this->fuckhtml
  1905. ->getElementsByClassName(
  1906. $this->findstyles(
  1907. [
  1908. "display" => "table-cell",
  1909. "vertical-align" => "middle",
  1910. "height" => "52px",
  1911. "text-align" => "center"
  1912. ],
  1913. self::is_class
  1914. ),
  1915. "a"
  1916. );
  1917. if(count($buttons) !== 0){
  1918. foreach($buttons as $button){
  1919. if(isset($button["attributes"]["href"])){
  1920. $sublink[
  1921. $this->fuckhtml
  1922. ->getTextContent(
  1923. $button
  1924. )
  1925. ] =
  1926. $this->unshiturl(
  1927. $button["attributes"]["href"]
  1928. );
  1929. }
  1930. }
  1931. }
  1932. }
  1933. // append description_after (contains carousel info)
  1934. $description = array_merge(
  1935. $description,
  1936. $description_after
  1937. );
  1938. $out["answer"][] = [
  1939. "title" =>
  1940. $this->fuckhtml
  1941. ->getTextContent(
  1942. $wiki_title[0]
  1943. ),
  1944. "description" => $description,
  1945. "url" => $url,
  1946. "thumb" => $image,
  1947. "table" => $table,
  1948. "sublink" => $sublink
  1949. ];
  1950. continue;
  1951. }
  1952. //
  1953. // Detect related searches containers
  1954. //
  1955. $container_title =
  1956. $this->fuckhtml
  1957. ->getElementsByClassName(
  1958. $this->findstyles(
  1959. [
  1960. "font-weight" => "bold",
  1961. "font-size" => "16px",
  1962. "color" => "#000",
  1963. "margin" => "0",
  1964. "padding" => "12px 16px 0 16px"
  1965. ],
  1966. self::is_class
  1967. ),
  1968. "div"
  1969. );
  1970. if(count($container_title) !== 0){
  1971. // get carousel entries
  1972. $carousels = $this->parsecarousels($container["innerHTML"]);
  1973. $this->fuckhtml->load($container);
  1974. foreach($carousels as $carousel){
  1975. foreach($carousel as $item){
  1976. if($item["url"] !== null){
  1977. $out["related"][] = $item["url"];
  1978. }
  1979. }
  1980. }
  1981. $container_title =
  1982. strtolower(
  1983. $this->fuckhtml
  1984. ->getTextContent(
  1985. $container_title[0]
  1986. )
  1987. );
  1988. switch($container_title){
  1989. case "related searches":
  1990. case "people also search for":
  1991. //
  1992. // Parse related searches
  1993. //
  1994. $as =
  1995. $this->fuckhtml
  1996. ->getElementsByTagName("a");
  1997. foreach($as as $a){
  1998. $out["related"][] =
  1999. $this->fuckhtml
  2000. ->getTextContent($a);
  2001. }
  2002. break;
  2003. case "people also ask":
  2004. // get related queries
  2005. $divs =
  2006. $this->fuckhtml
  2007. ->getElementsByTagName("div");
  2008. foreach($divs as $div){
  2009. // add accdef's here
  2010. if($has_appended_accdef === false){
  2011. $out["web"] = array_merge($out["web"], $accdefs);
  2012. $has_appended_accdef = true;
  2013. }
  2014. // add accdef's questions
  2015. if(isset($div["attributes"]["role"])){
  2016. $out["related"][] =
  2017. $this->fuckhtml
  2018. ->getTextContent($div);
  2019. continue;
  2020. }
  2021. }
  2022. break;
  2023. }
  2024. continue;
  2025. }
  2026. //
  2027. // Parse news
  2028. //
  2029. $title =
  2030. $this->fuckhtml
  2031. ->getElementsByClassName(
  2032. $this->findstyles(
  2033. [
  2034. "font-size" => "16px",
  2035. "line-height" => "20px",
  2036. "font-weight" => "400"
  2037. ],
  2038. self::is_class
  2039. ),
  2040. "div"
  2041. );
  2042. if(count($title) !== 0){
  2043. $carousels = $this->parsecarousels();
  2044. $this->fuckhtml->load($container);
  2045. if(count($carousels) === 0){
  2046. // no carousels found
  2047. continue;
  2048. }
  2049. $title =
  2050. strtolower(
  2051. $this->fuckhtml
  2052. ->getTextContent(
  2053. $title[0]
  2054. )
  2055. );
  2056. if(
  2057. preg_match(
  2058. '/^latest from|^top stories/',
  2059. $title
  2060. )
  2061. ){
  2062. // Found news article
  2063. foreach($carousels[0] as $carousel){
  2064. if($carousel["image"] !== null){
  2065. $thumb = [
  2066. "url" => $carousel["image"],
  2067. "ratio" => "16:9"
  2068. ];
  2069. }else{
  2070. $thumb = [
  2071. "url" => null,
  2072. "ratio" => null
  2073. ];
  2074. }
  2075. $out["news"][] = [
  2076. "title" => $carousel["title"],
  2077. "description" => $carousel["description"],
  2078. "date" => $carousel["date"],
  2079. "thumb" => $thumb,
  2080. "url" => $carousel["url"]
  2081. ];
  2082. }
  2083. }
  2084. elseif(
  2085. $title == "images"
  2086. ){
  2087. foreach($carousels as $carousel){
  2088. foreach($carousel as $item){
  2089. $out["image"][] = [
  2090. "title" => $item["title"],
  2091. "source" => [
  2092. [
  2093. "url" => $item["url"],
  2094. "width" => $item["image_width"],
  2095. "height" => $item["image_height"]
  2096. ],
  2097. [
  2098. "url" => $item["image"],
  2099. "width" => $item["thumb_width"],
  2100. "height" => $item["thumb_height"]
  2101. ]
  2102. ],
  2103. "url" => $item["ref"]
  2104. ];
  2105. }
  2106. }
  2107. }
  2108. continue;
  2109. }
  2110. //
  2111. // Detect nodes with only text + links
  2112. //
  2113. // ignore elements with <style> tags
  2114. $style =
  2115. $this->fuckhtml
  2116. ->getElementsByTagName("style");
  2117. if(count($style) !== 0){
  2118. continue;
  2119. }
  2120. $as =
  2121. $this->fuckhtml
  2122. ->getElementsByTagName("a");
  2123. $description = [];
  2124. foreach($as as $a){
  2125. //
  2126. // Detect next page
  2127. //
  2128. if(
  2129. isset($a["attributes"]["aria-label"]) &&
  2130. strtolower($a["attributes"]["aria-label"]) == "next page"
  2131. ){
  2132. $out["npt"] =
  2133. $this->backend->store(
  2134. $this->fuckhtml
  2135. ->getTextContent(
  2136. $a["attributes"]["href"]
  2137. ),
  2138. $pagetype,
  2139. $ip
  2140. );
  2141. continue 2;
  2142. }
  2143. //
  2144. // Parse as text node
  2145. //
  2146. $container["innerHTML"] =
  2147. explode(
  2148. $a["outerHTML"],
  2149. $container["innerHTML"],
  2150. 2
  2151. );
  2152. $before =
  2153. $this->fuckhtml
  2154. ->getTextContent(
  2155. $container["innerHTML"][0],
  2156. false,
  2157. false
  2158. );
  2159. // set after
  2160. if(count($container["innerHTML"]) === 2){
  2161. $container["innerHTML"] =
  2162. $container["innerHTML"][1];
  2163. }else{
  2164. $container["innerHTML"] = "";
  2165. }
  2166. if($before != ""){
  2167. $description[] = [
  2168. "type" => "text",
  2169. "value" => $before
  2170. ];
  2171. }
  2172. // add link
  2173. $description[] = [
  2174. "type" => "link",
  2175. "url" =>
  2176. $this->unshiturl(
  2177. $a["attributes"]
  2178. ["href"]
  2179. ),
  2180. "value" =>
  2181. $this->fuckhtml
  2182. ->getTextContent(
  2183. $a
  2184. )
  2185. ];
  2186. }
  2187. if($container["innerHTML"] != ""){
  2188. $description[] = [
  2189. "type" => "text",
  2190. "value" =>
  2191. $this->fuckhtml
  2192. ->getTextContent(
  2193. $container["innerHTML"]
  2194. )
  2195. ];
  2196. }
  2197. $out["answer"][] = [
  2198. "title" => "Notice",
  2199. "description" => $description,
  2200. "url" => null,
  2201. "thumb" => null,
  2202. "table" => [],
  2203. "sublink" => []
  2204. ];
  2205. }
  2206. //
  2207. // remove duplicate web links cause instant answers
  2208. // sometimes contains duplicates
  2209. //
  2210. $c = count($out["web"]);
  2211. $links = [];
  2212. for($i=0; $i<$c; $i++){
  2213. foreach($links as $link){
  2214. if($out["web"][$i]["url"] == $link){
  2215. unset($out["web"][$i]);
  2216. continue 2;
  2217. }
  2218. }
  2219. $links[] = $out["web"][$i]["url"];
  2220. }
  2221. $out["web"] = array_values($out["web"]);
  2222. return $out;
  2223. }
  2224. public function image($get){
  2225. // generate parameters
  2226. if($get["npt"]){
  2227. [$params, $proxy] =
  2228. $this->backend->get(
  2229. $get["npt"],
  2230. "images"
  2231. );
  2232. $params = json_decode($params, true);
  2233. }else{
  2234. $search = $get["s"];
  2235. if(strlen($search) === 0){
  2236. throw new Exception("Search term is empty!");
  2237. }
  2238. $proxy = $this->backend->get_ip();
  2239. $country = $get["country"];
  2240. $nsfw = $get["nsfw"];
  2241. $lang = $get["lang"];
  2242. $time = $get["time"];
  2243. $size = $get["size"];
  2244. $ratio = $get["ratio"];
  2245. $color = $get["color"];
  2246. $type = $get["type"];
  2247. $format = $get["format"];
  2248. $rights = $get["rights"];
  2249. $params = [
  2250. "q" => $search,
  2251. "tbm" => "isch"
  2252. ];
  2253. // country
  2254. if($country != "any"){
  2255. $params["gl"] = $country;
  2256. }
  2257. // nsfw
  2258. $params["safe"] = $nsfw == "yes" ? "off" : "active";
  2259. // language
  2260. if($lang != "any"){
  2261. $params["lr"] = "lang_" . $lang;
  2262. }
  2263. $tbs = [];
  2264. // time
  2265. if($time != "any"){
  2266. $tbs[] = "qrd:" . $time;
  2267. }
  2268. // size
  2269. if($size != "any"){
  2270. if(
  2271. in_array(
  2272. $size,
  2273. ["l", "s", "i"]
  2274. )
  2275. ){
  2276. $tbs[] = "isz:" . $size;
  2277. }else{
  2278. $tbs[] = "tbz:lt";
  2279. $tbs[] = "islt:" . $size;
  2280. }
  2281. }
  2282. // ratio
  2283. if($ratio != "any"){
  2284. $tbs[] = "iar:" . $ratio;
  2285. }
  2286. // color
  2287. if($color != "any"){
  2288. if(
  2289. in_array(
  2290. $color,
  2291. ["color", "gray", "trans"]
  2292. )
  2293. ){
  2294. $tbs[] = "ic:" . $color;
  2295. }else{
  2296. $tbs[] = "ic:specific";
  2297. $tbs[] = "isc:" . $color;
  2298. }
  2299. }
  2300. // type
  2301. if($type != "any"){
  2302. $tbs[] = "itp:" . $type;
  2303. }
  2304. // format
  2305. if($format != "any"){
  2306. $tbs[] = "ift:" . $format;
  2307. }
  2308. // rights
  2309. if($rights != "any"){
  2310. $tbs[] = "il:" . $rights;
  2311. }
  2312. // append tbs
  2313. if(count($tbs) !== 0){
  2314. $params["tbs"] =
  2315. implode(",", $tbs);
  2316. }
  2317. }
  2318. /*
  2319. $handle = fopen("scraper/google-img.html", "r");
  2320. $html = fread($handle, filesize("scraper/google-img.html"));
  2321. fclose($handle);*/
  2322. // scrape images
  2323. try{
  2324. $html =
  2325. $this->get(
  2326. $proxy,
  2327. "https://www.google.com/search",
  2328. $params
  2329. );
  2330. }catch(Exception $error){
  2331. throw new Exception("Failed to get search page");
  2332. }
  2333. if($this->detect_sorry($html)){
  2334. throw new Exception("Google blocked this 4get instance. Please set up a proxy!");
  2335. }
  2336. $out = [
  2337. "status" => "ok",
  2338. "npt" => null,
  2339. "image" => []
  2340. ];
  2341. $images =
  2342. $this->fuckhtml
  2343. ->getElementsByClassName(
  2344. "islrtb isv-r",
  2345. "div"
  2346. );
  2347. foreach($images as $image){
  2348. $this->fuckhtml->load($image);
  2349. $img =
  2350. $this->fuckhtml
  2351. ->getElementsByTagName("img")[0];
  2352. $og_width = (int)$image["attributes"]["data-ow"];
  2353. $og_height = (int)$image["attributes"]["data-oh"];
  2354. $thumb_width = (int)$image["attributes"]["data-tw"];
  2355. $ratio = $og_width / $og_height;
  2356. if(isset($img["attributes"]["data-src"])){
  2357. $src = &$img["attributes"]["data-src"];
  2358. }else{
  2359. $src = &$img["attributes"]["src"];
  2360. }
  2361. $thumb_height = floor($thumb_width / $ratio);
  2362. $out["image"][] = [
  2363. "title" =>
  2364. $this->titledots(
  2365. $this->fuckhtml
  2366. ->getTextContent(
  2367. $image["attributes"]["data-pt"]
  2368. )
  2369. ),
  2370. "source" => [
  2371. [
  2372. "url" =>
  2373. $this->fuckhtml
  2374. ->getTextContent(
  2375. $image["attributes"]["data-ou"]
  2376. ),
  2377. "width" => $og_width,
  2378. "height" => $og_height
  2379. ],
  2380. [
  2381. "url" =>
  2382. $this->fuckhtml
  2383. ->getTextContent(
  2384. $src
  2385. ),
  2386. "width" => $thumb_width,
  2387. "height" => $thumb_height
  2388. ]
  2389. ],
  2390. "url" =>
  2391. $this->fuckhtml
  2392. ->getTextContent(
  2393. $image["attributes"]["data-ru"]
  2394. )
  2395. ];
  2396. }
  2397. // get next page
  2398. // https://www.google.com/search
  2399. // ?q=higurashi
  2400. // &tbm=isch
  2401. // &async=_id%3Aislrg_c%2C_fmt%3Ahtml
  2402. // &asearch=ichunklite
  2403. // &ved=0ahUKEwidjYXJqJSAAxWrElkFHZ07CDwQtDIIQygA
  2404. if(count($out["image"]) !== 100){
  2405. // no more results
  2406. return $out;
  2407. }
  2408. if($get["npt"]){
  2409. // update nextpage information
  2410. $params["start"] = (int)$params["start"] + count($out["image"]);
  2411. $params["ijn"] = (int)$params["ijn"] + 1;
  2412. $out["npt"] =
  2413. $this->backend->store(
  2414. json_encode($params),
  2415. "images",
  2416. $proxy
  2417. );
  2418. }else{
  2419. // scrape nextpage information
  2420. $this->fuckhtml->load($html);
  2421. $ved =
  2422. $this->fuckhtml
  2423. ->getElementById("islrg", "div");
  2424. if($ved){
  2425. $ved =
  2426. $this->fuckhtml
  2427. ->getTextContent(
  2428. $ved["attributes"]["data-ved"]
  2429. );
  2430. // &vet=1{$ved}..i (10ahUKEwidjYXJqJSAAxWrElkFHZ07CDwQtDIIQygA..i)
  2431. /*
  2432. These 2 are handled by us
  2433. start = start + number of results
  2434. ijn = current page number
  2435. */
  2436. // &start=100
  2437. // &ijn=1
  2438. // &imgvl=CAEY7gQgBSj3Aji8VTjXVUC4AUC3AUgAYNdV
  2439. preg_match(
  2440. '/var e=\'([A-z0-9]+)\';/',
  2441. $html,
  2442. $imgvl
  2443. );
  2444. if(isset($imgvl[1])){
  2445. $imgvl = $imgvl[1];
  2446. $params["async"] = "_id:islrg_c,_fmt:html";
  2447. $params["asearch"] = "ichunklite";
  2448. $params["ved"] = $ved;
  2449. $params["vet"] = "1" . $ved . "..i";
  2450. $params["start"] = 100;
  2451. $params["ijn"] = 1;
  2452. $params["imgvl"] = $imgvl;
  2453. $out["npt"] =
  2454. $this->backend->store(
  2455. json_encode($params),
  2456. "images",
  2457. $proxy
  2458. );
  2459. }
  2460. }
  2461. }
  2462. return $out;
  2463. }
  2464. private function hms2int($time){
  2465. $parts = explode(":", $time, 3);
  2466. $time = 0;
  2467. if(count($parts) === 3){
  2468. // hours
  2469. $time = $time + ((int)$parts[0] * 3600);
  2470. array_shift($parts);
  2471. }
  2472. if(count($parts) === 2){
  2473. // minutes
  2474. $time = $time + ((int)$parts[0] * 60);
  2475. array_shift($parts);
  2476. }
  2477. // seconds
  2478. $time = $time + (int)$parts[0];
  2479. return $time;
  2480. }
  2481. private function parsejavascript($html){
  2482. $this->fuckhtml->load($html);
  2483. $styles =
  2484. $this->fuckhtml
  2485. ->getElementsByTagName("style");
  2486. $this->computedstyle = [];
  2487. $this->ask = [];
  2488. foreach($styles as $style){
  2489. $this->computedstyle =
  2490. array_merge(
  2491. $this->computedstyle,
  2492. $this->parsestyles($style["innerHTML"])
  2493. );
  2494. }
  2495. // get images in javascript var
  2496. preg_match(
  2497. '/google\.ldi=({[^}]+})/',
  2498. $html,
  2499. $this->js_image
  2500. );
  2501. if(count($this->js_image) !== 0){
  2502. $this->js_image = json_decode($this->js_image[1], true);
  2503. }else{
  2504. $this->js_image = [];
  2505. }
  2506. // additional js_images present in <script> tags
  2507. // ugh i fucking hate you
  2508. $scripts =
  2509. $this->fuckhtml
  2510. ->getElementsByTagName("script");
  2511. foreach($scripts as $script){
  2512. if(!isset($script["innerHTML"])){
  2513. continue;
  2514. }
  2515. preg_match_all(
  2516. '/var s=\'(data:image[^\']+)\';var i=\[(\'[^\;]*\')];/',
  2517. $script["innerHTML"],
  2518. $image_grep
  2519. );
  2520. if(count($image_grep[0]) !== 0){
  2521. $items = explode(",", $image_grep[2][0]);
  2522. $value =
  2523. $this->fuckhtml
  2524. ->getTextContent(
  2525. $image_grep[1][0]
  2526. );
  2527. foreach($items as $item){
  2528. $this->js_image[trim($item, "' ")] = $value;
  2529. }
  2530. }
  2531. // even more javascript crap
  2532. // "People also ask" node is loaded trough javascript
  2533. preg_match_all(
  2534. '/window\.jsl\.dh\(\'([^\']+)\',\'(.+)\'\);/',
  2535. $script["innerHTML"],
  2536. $ask_grep
  2537. );
  2538. for($i=0; $i<count($ask_grep[0]); $i++){
  2539. $this->ask[trim($ask_grep[1][$i])] =
  2540. $this->fuckhtml->parseJsString(
  2541. $ask_grep[2][$i]
  2542. );
  2543. }
  2544. }
  2545. }
  2546. private function findstyles($rules, $is){
  2547. ksort($rules);
  2548. foreach($this->computedstyle as $stylename => $styles){
  2549. if($styles == $rules){
  2550. preg_match(
  2551. '/\\' . $is . '([^ .]+)/',
  2552. $stylename,
  2553. $out
  2554. );
  2555. if(count($out) === 2){
  2556. return $out[1];
  2557. }
  2558. return false;
  2559. }
  2560. }
  2561. return false;
  2562. }
  2563. private function parsestyles($style){
  2564. // get style tags
  2565. preg_match_all(
  2566. '/([^{]+){([^}]+)}/',
  2567. $style,
  2568. $tags_regex
  2569. );
  2570. $tags = [];
  2571. for($i=0; $i<count($tags_regex[0]); $i++){
  2572. $tagnames = explode(",", trim($tags_regex[1][$i]));
  2573. foreach($tagnames as $tagname){
  2574. $tagname = trim($tagname);
  2575. if(!isset($tags[$tagname])){
  2576. $tags[$tagname] = [];
  2577. }
  2578. $values = explode(";", $tags_regex[2][$i]);
  2579. foreach($values as $value){
  2580. $value = explode(":", $value, 2);
  2581. if(count($value) !== 2){
  2582. continue;
  2583. }
  2584. $tags[$tagname][trim($value[0])] =
  2585. trim($value[1]);
  2586. }
  2587. }
  2588. }
  2589. foreach($tags as &$value){
  2590. ksort($value);
  2591. }
  2592. return $tags;
  2593. }
  2594. private function getimage($id){
  2595. if(isset($this->js_image[$id])){
  2596. $return = $this->fuckhtml->parseJsString($this->js_image[$id]);
  2597. if(
  2598. $return != "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAUCAYAAACNiR0NAAABAUlEQVR4AWMYesChoYElLjkzPj4lY3d8csZjIL4MxPNjUzPcSTYsISFLAqj5NBD/h+LPQPwbiT87NCuLh2gDgRr2QzXuT0jNMoBYksARn5zuHJ+UcR0kB6RXE2VYXHJGOlTDZmzyIJcB5e+D1CSkZDgQNBAaZv+jU1JkcKpJygiGeZ0I76a/Byq8jU9NZFqaCNTA48SE33/iDcw8TIyBt0GKQTFN0Msp6f2EIyUpo57YSIlLSrMhIg0WCIBcCfXSdlzJBsheTHQ6jEnOUgEFOLaEDbMIlhZBOYrorAdJk+nroVnvPsSgdGdoOF7HZyhZ2XPoGQoqjbCpIbt0AiejIQMArVLI7k/DXFkAAAAASUVORK5CYII=" &&
  2599. $return != "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABgAAAAYCAYAAADgdz34AAAA6ElEQVR4Ae2UvQ7BYBSGW4uFxlVICLsYDA0D14NEunRxHSJ+BkYG9ibiHtgkuvpZWOod3uFESqpOF/ElT/q958h5OtQx/iexY/evY9ACJjBewUtkXHEPyBYUNQQuCETuggNrF2DHF3A4kfUMmLB+BoUYAg4nIX0TTNnbfCjg8HDBGuR4z4Ij+813giAC8rcrkXusjdQEpMpcYt5rCzrMaea7tqD9JLhpCyrMZeadpmApssPaUOszXQALGHz67De0/2gpMGPP014VFpizfgJ5zWXnAF8MryW1rj3x5l8LJANQF1lZQH5f8AAWpNcUs6HAEAAAAABJRU5ErkJggg=="
  2600. ){
  2601. if(
  2602. preg_match(
  2603. '/^\/\//',
  2604. $return
  2605. )
  2606. ){
  2607. return 'https:' . $return;
  2608. }
  2609. return $return;
  2610. }
  2611. return null;
  2612. }
  2613. }
  2614. private function parsecarousels(&$item_to_remove = false){
  2615. $carousels =
  2616. $this->fuckhtml
  2617. ->getElementsByClassName(
  2618. $this->findstyles(
  2619. [
  2620. "padding" => "16px",
  2621. "position" => "relative"
  2622. ],
  2623. self::is_class
  2624. )
  2625. );
  2626. $return = [];
  2627. for($i=0; $i<count($carousels); $i++){
  2628. if(!isset($carousels[$i]["outerHTML"])){
  2629. continue;
  2630. }
  2631. $this->fuckhtml->load($carousels[$i]);
  2632. if($item_to_remove !== false){
  2633. $item_to_remove =
  2634. str_replace(
  2635. $carousels[$i]["outerHTML"],
  2636. "",
  2637. $item_to_remove
  2638. );
  2639. }
  2640. $pcitems =
  2641. $this->fuckhtml
  2642. ->getElementsByClassName(
  2643. "pcitem",
  2644. "div"
  2645. );
  2646. foreach($pcitems as $pcitem){
  2647. $this->fuckhtml->load($pcitem);
  2648. $out = [
  2649. "url" => null,
  2650. "ref" => null,
  2651. "image" => null,
  2652. "thumb_width" => null,
  2653. "thumb_height" => null,
  2654. "image_width" => null,
  2655. "image_height" => null,
  2656. "title" => null,
  2657. "description" => null,
  2658. "subtext" => null,
  2659. "date" => null
  2660. ];
  2661. $url =
  2662. $this->unshiturl(
  2663. $this->fuckhtml
  2664. ->getElementsByTagName("a")
  2665. [0]
  2666. ["attributes"]
  2667. ["href"],
  2668. true
  2669. );
  2670. // set ref
  2671. $out["ref"] = $url["ref"];
  2672. // set url
  2673. $out["url"] = $url["url"];
  2674. // set sizes
  2675. $out["thumb_width"] = $url["thumb_width"];
  2676. $out["thumb_height"] = $url["thumb_height"];
  2677. $out["image_width"] = $url["image_width"];
  2678. $out["image_height"] = $url["image_height"];
  2679. // get image
  2680. $out["image"] =
  2681. $this->fuckhtml
  2682. ->getElementsByTagName(
  2683. "img"
  2684. );
  2685. if(count($out["image"]) !== 0){
  2686. // get title from image
  2687. if(isset($out["image"][0]["attributes"]["alt"])){
  2688. $out["title"] =
  2689. $this->titledots(
  2690. $this->fuckhtml
  2691. ->getTextContent(
  2692. $out["image"][0]["attributes"]["alt"]
  2693. )
  2694. );
  2695. }
  2696. // get image url
  2697. if(isset($out["image"][0]["attributes"]["id"])){
  2698. $out["image"] = $this->getimage($out["image"][0]["attributes"]["id"]);
  2699. }
  2700. elseif(isset($out["image"][0]["attributes"]["data-ll"])){
  2701. $out["image"] =
  2702. $this->fuckhtml
  2703. ->getTextContent(
  2704. $out["image"][0]["attributes"]["data-ll"]
  2705. );
  2706. }else{
  2707. // failed to get image information
  2708. $out["image"] = null;
  2709. }
  2710. if($out["image"] == 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABgAAAAYAgMAAACdGdVrAAAADFBMVEVMaXFChfRChfRChfT0tCPZAAAAA3RSTlMAgFJEkGxNAAAAL0lEQVR4AWPADxgdwBT3BTDF9AUiuhdC6WNK/v///y+UggrClSA07EWVglmEFwAA5eYSExeCwigAAAAASUVORK5CYII='){
  2711. // found arrow image base64, skip entry
  2712. continue;
  2713. }
  2714. }else{
  2715. // Could not find any image in node
  2716. $out["image"] = null;
  2717. }
  2718. // get title from spans
  2719. $title =
  2720. $this->fuckhtml
  2721. ->getElementsByClassName(
  2722. $this->findstyles(
  2723. [
  2724. "color" => "#1967d2"
  2725. ],
  2726. self::is_class
  2727. ),
  2728. "span"
  2729. );
  2730. if(count($title) !== 0){
  2731. $out["title"] =
  2732. $this->fuckhtml
  2733. ->getTextContent(
  2734. $title[0]
  2735. );
  2736. }
  2737. // get textnodes
  2738. $textnodes =
  2739. $this->fuckhtml
  2740. ->getElementsByClassName(
  2741. $this->findstyles(
  2742. [
  2743. "white-space" => "pre-line",
  2744. "word-wrap" => "break-word"
  2745. ],
  2746. self::is_class
  2747. )
  2748. );
  2749. $subtext = null;
  2750. if(count($textnodes) !== 0){
  2751. // get date
  2752. $date =
  2753. $this->fuckhtml
  2754. ->getTextContent(
  2755. $textnodes[count($textnodes) - 1],
  2756. true
  2757. );
  2758. if(str_replace("\n", " ", $date) == $title){
  2759. $date = null;
  2760. }else{
  2761. if(strpos($date, "\n") !== false){
  2762. $date = explode("\n", $date);
  2763. $date = $date[count($date) - 1];
  2764. }
  2765. elseif(strpos($date, "•") !== false){
  2766. $date = explode("•", $date);
  2767. $date = ltrim($date[count($date) - 1]);
  2768. }else{
  2769. $date = null;
  2770. }
  2771. }
  2772. if($date !== null){
  2773. $date = strtotime($date);
  2774. }
  2775. // get description
  2776. $description =
  2777. $this->fuckhtml
  2778. ->getTextContent(
  2779. $textnodes[0]
  2780. );
  2781. if($out["title"] === null){
  2782. if($date === null){
  2783. $out["title"] = $description;
  2784. $description = null;
  2785. }else{
  2786. $out["title"] = parse_url($out["url"], PHP_URL_HOST);
  2787. }
  2788. }
  2789. if(isset($textnodes[1])){
  2790. $out["subtext"] =
  2791. $this->fuckhtml
  2792. ->getTextContent(
  2793. $textnodes[1]
  2794. );
  2795. }
  2796. }else{
  2797. $date = null;
  2798. $description = null;
  2799. }
  2800. $out["date"] = $date;
  2801. $out["description"] = $this->titledots($description);
  2802. if($out["url"] === null){
  2803. $out["url"] = $out["title"];
  2804. }
  2805. if($out["title"] == $out["description"]){
  2806. $out["description"] = null;
  2807. }
  2808. $return[$i][] = $out;
  2809. }
  2810. }
  2811. return $return;
  2812. }
  2813. private function unshiturl($url, $return_size = false){
  2814. // get parameters from URL
  2815. $url =
  2816. $this->fuckhtml
  2817. ->getTextContent($url);
  2818. $newurl = parse_url($url, PHP_URL_QUERY);
  2819. if($newurl == ""){
  2820. // probably telephone number
  2821. return $url;
  2822. }
  2823. $url = $newurl;
  2824. unset($newurl);
  2825. parse_str($url, $query);
  2826. if(isset($query["imgurl"])){
  2827. $url = $query["imgurl"];
  2828. }
  2829. elseif(isset($query["q"])){
  2830. $url = $query["q"];
  2831. }
  2832. // rewrite URLs to remove extra tracking parameters
  2833. $domain = parse_url($url, PHP_URL_HOST);
  2834. if(
  2835. preg_match(
  2836. '/wikipedia.org$/',
  2837. $domain
  2838. )
  2839. ){
  2840. // rewrite wikipedia mobile URLs to desktop
  2841. $url =
  2842. $this->replacedomain(
  2843. $url,
  2844. preg_replace(
  2845. '/([a-z0-9]+)(\.m\.)/',
  2846. '$1.',
  2847. $domain
  2848. )
  2849. );
  2850. }
  2851. elseif(
  2852. preg_match(
  2853. '/imdb\.com$|youtube\.[^.]+$/',
  2854. $domain
  2855. )
  2856. ){
  2857. // rewrite imdb and youtube mobile URLs too
  2858. $url =
  2859. $this->replacedomain(
  2860. $url,
  2861. preg_replace(
  2862. '/^m\./',
  2863. "",
  2864. $domain
  2865. )
  2866. );
  2867. }
  2868. elseif(
  2869. preg_match(
  2870. '/play\.google\.[^.]+$/',
  2871. $domain
  2872. )
  2873. ){
  2874. // remove referrers from play.google.com
  2875. $oldquery = parse_url($url, PHP_URL_QUERY);
  2876. if($oldquery !== null){
  2877. parse_str($oldquery, $query);
  2878. if(isset($query["referrer"])){ unset($query["referrer"]); }
  2879. if(isset($query["hl"])){ unset($query["hl"]); }
  2880. if(isset($query["gl"])){ unset($query["gl"]); }
  2881. $query = http_build_query($query);
  2882. $url =
  2883. str_replace(
  2884. $oldquery,
  2885. $query,
  2886. $url
  2887. );
  2888. }
  2889. }
  2890. elseif(
  2891. preg_match(
  2892. '/twitter\.com$/',
  2893. $domain
  2894. )
  2895. ){
  2896. // remove more referrers from twitter.com
  2897. $oldquery = parse_url($url, PHP_URL_QUERY);
  2898. if($oldquery !== null){
  2899. parse_str($oldquery, $query);
  2900. if(isset($query["ref_src"])){ unset($query["ref_src"]); }
  2901. $query = http_build_query($query);
  2902. if($query != ""){
  2903. $query .= "?" . $query;
  2904. }
  2905. $url =
  2906. str_replace(
  2907. '?' . $oldquery,
  2908. $query,
  2909. $url
  2910. );
  2911. }
  2912. }
  2913. elseif(
  2914. preg_match(
  2915. '/maps\.google\.[^.]+/',
  2916. $domain
  2917. )
  2918. ){
  2919. if(stripos($url, "maps?") !== false){
  2920. //https://maps.google.com/maps?daddr=Johnny,+603+Rue+St+Georges,+Saint-J%C3%A9r%C3%B4me,+Quebec+J7Z+5B7
  2921. $query = parse_url($url, PHP_URL_QUERY);
  2922. if($query !== null){
  2923. parse_str($query, $query);
  2924. if(isset($query["daddr"])){
  2925. $url =
  2926. "https://maps.google.com/maps?daddr=" .
  2927. urlencode($query["daddr"]);
  2928. }
  2929. }
  2930. }
  2931. }
  2932. if($return_size){
  2933. return [
  2934. "url" => $url,
  2935. "ref" => isset($query["imgrefurl"]) ? $query["imgrefurl"] : null,
  2936. "thumb_width" => isset($query["tbnw"]) ? (int)$query["tbnw"] : null,
  2937. "thumb_height" => isset($query["tbnh"]) ? (int)$query["tbnh"] : null,
  2938. "image_width" => isset($query["w"]) ? (int)$query["w"] : null,
  2939. "image_height" => isset($query["h"]) ? (int)$query["h"] : null
  2940. ];
  2941. }
  2942. return $url;
  2943. }
  2944. private function replacedomain($url, $domain){
  2945. return
  2946. preg_replace(
  2947. '/(https?:\/\/)([^\/]+)/',
  2948. '$1' . $domain,
  2949. $url
  2950. );
  2951. }
  2952. private function titledots($title){
  2953. return rtrim($title, ". \t\n\r\0\x0B");
  2954. }
  2955. private function detect_sorry($html){
  2956. $this->fuckhtml->load($html);
  2957. $detect_sorry =
  2958. $this->fuckhtml
  2959. ->getElementsByTagName("title");
  2960. if(
  2961. isset($detect_sorry[0]) &&
  2962. $detect_sorry[0]["innerHTML"] == "302 Moved"
  2963. ){
  2964. return true;
  2965. }
  2966. return false;
  2967. }
  2968. }