Revert "src/Tweeper.php: fix rendering Instagram images in some feed readers"

[tweeper.git] / src / Tweeper.php

diff --git a/src/Tweeper.php b/src/Tweeper.php
index e98623b..f79bb08 100644 (file)
--- a/src/Tweeper.php
+++ b/src/Tweeper.php
@@ -6,7 +6,7 @@ namespace Tweeper;
  * @file
  * Tweeper - a Twitter to RSS web scraper.
  *
  * @file
  * Tweeper - a Twitter to RSS web scraper.
  *

- * Copyright (C) 2013-2018  Antonio Ospite <ao2@ao2.it>
+ * Copyright (C) 2013-2020  Antonio Ospite <ao2@ao2.it>

  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by


@@ -36,7 +36,7 @@ date_default_timezone_set('UTC');
  */
 class Tweeper {
 
  */
 class Tweeper {
 

-  private static $userAgent = "Mozilla/5.0";
+  private static $userAgent = "APIs-Google (+https://developers.google.com/webmasters/APIs-Google.html)";

   private static $maxConnectionTimeout = 5;
   private static $maxConnectionRetries = 5;
 
   private static $maxConnectionTimeout = 5;
   private static $maxConnectionRetries = 5;
 


@@ -111,24 +111,35 @@ class Tweeper {
       }
     } while (curl_errno($ch) == CURLE_OPERATION_TIMEDOUT && ++$attempt < Tweeper::$maxConnectionRetries);
 
       }
     } while (curl_errno($ch) == CURLE_OPERATION_TIMEDOUT && ++$attempt < Tweeper::$maxConnectionRetries);
 

+    $response_code = curl_getinfo($ch, CURLINFO_RESPONSE_CODE);
+    if (FALSE === $response_code) {
+      trigger_error(curl_error($ch), E_USER_WARNING);
+      return FALSE;
+    }
+
+    if ($response_code >= 400) {
+      trigger_error("HTTP reponse code $response_code", E_USER_WARNING);
+      return FALSE;
+    }
+

     return $ret;
   }
 
   /**
    * Get the contents from a URL.
    */
     return $ret;
   }
 
   /**
    * Get the contents from a URL.
    */

-  private static function getUrlContents($url) {
+  private static function getUrlContents($url, $user_agent = NULL) {

     $ch = curl_init($url);
     $ch = curl_init($url);

-    curl_setopt_array($ch, array(
+    curl_setopt_array($ch, [

       CURLOPT_HEADER => FALSE,
       CURLOPT_CONNECTTIMEOUT => Tweeper::$maxConnectionTimeout,
       // Follow http redirects to get the real URL.
       CURLOPT_FOLLOWLOCATION => TRUE,
       CURLOPT_COOKIEFILE => "",
       CURLOPT_RETURNTRANSFER => TRUE,
       CURLOPT_HEADER => FALSE,
       CURLOPT_CONNECTTIMEOUT => Tweeper::$maxConnectionTimeout,
       // Follow http redirects to get the real URL.
       CURLOPT_FOLLOWLOCATION => TRUE,
       CURLOPT_COOKIEFILE => "",
       CURLOPT_RETURNTRANSFER => TRUE,

-      CURLOPT_HTTPHEADER => array('Accept-language: en'),
-      CURLOPT_USERAGENT => Tweeper::$userAgent,
-    ));
+      CURLOPT_HTTPHEADER => ['Accept-language: en'],
+      CURLOPT_USERAGENT => isset($user_agent) ? $user_agent : Tweeper::$userAgent,
+    ]);

     $contents = Tweeper::curlExec($ch);
     curl_close($ch);
 
     $contents = Tweeper::curlExec($ch);
     curl_close($ch);
 


@@ -138,17 +149,17 @@ class Tweeper {
   /**
    * Get the headers from a URL.
    */
   /**
    * Get the headers from a URL.
    */

-  private static function getUrlInfo($url) {
+  private static function getUrlInfo($url, $user_agent = NULL) {

     $ch = curl_init($url);
     $ch = curl_init($url);

-    curl_setopt_array($ch, array(
+    curl_setopt_array($ch, [

       CURLOPT_HEADER => TRUE,
       CURLOPT_NOBODY => TRUE,
       CURLOPT_CONNECTTIMEOUT => Tweeper::$maxConnectionTimeout,
       // Follow http redirects to get the real URL.
       CURLOPT_FOLLOWLOCATION => TRUE,
       CURLOPT_RETURNTRANSFER => TRUE,
       CURLOPT_HEADER => TRUE,
       CURLOPT_NOBODY => TRUE,
       CURLOPT_CONNECTTIMEOUT => Tweeper::$maxConnectionTimeout,
       // Follow http redirects to get the real URL.
       CURLOPT_FOLLOWLOCATION => TRUE,
       CURLOPT_RETURNTRANSFER => TRUE,

-      CURLOPT_USERAGENT => Tweeper::$userAgent,
-    ));
+      CURLOPT_USERAGENT => isset($user_agent) ? $user_agent : Tweeper::$userAgent,
+    ]);

 
     $ret = Tweeper::curlExec($ch);
     if (FALSE === $ret) {
 
     $ret = Tweeper::curlExec($ch);
     if (FALSE === $ret) {


@@ -169,7 +180,7 @@ class Tweeper {
    * Generate an RSS <enclosure/> element.
    */
   public static function generateEnclosure($url) {
    * Generate an RSS <enclosure/> element.
    */
   public static function generateEnclosure($url) {

-    $supported_content_types = array(
+    $supported_content_types = [

       "application/octet-stream",
       "application/ogg",
       "application/pdf",
       "application/octet-stream",
       "application/ogg",
       "application/pdf",


@@ -188,7 +199,7 @@ class Tweeper {
       "video/mp4",
       "video/mpeg",
       "video/ogg",
       "video/mp4",
       "video/mpeg",
       "video/ogg",

-    );
+    ];

 
     $url_info = Tweeper::getUrlInfo($url);
     if (FALSE === $url_info) {
 
     $url_info = Tweeper::getUrlInfo($url);
     if (FALSE === $url_info) {


@@ -270,13 +281,13 @@ class Tweeper {
 
     $encoder = new XmlEncoder();
     $normalizer = new ObjectNormalizer();
 
     $encoder = new XmlEncoder();
     $normalizer = new ObjectNormalizer();

-    $serializer = new Serializer(array($normalizer), array($encoder));
+    $serializer = new Serializer([$normalizer], [$encoder]);

 
 

-    $serializer_options = array(
+    $serializer_options = [

       'xml_encoding' => "UTF-8",
       'xml_format_output' => TRUE,
       'xml_root_node_name' => $root_node_name,
       'xml_encoding' => "UTF-8",
       'xml_format_output' => TRUE,
       'xml_root_node_name' => $root_node_name,

-    );
+    ];

 
     $xml_data = $serializer->serialize($data, 'xml', $serializer_options);
     if (!$xml_data) {
 
     $xml_data = $serializer->serialize($data, 'xml', $serializer_options);
     if (!$xml_data) {


@@ -301,13 +312,19 @@ class Tweeper {
 
     $data = json_decode($matches[1], $assoc = TRUE);
 
 
     $data = json_decode($matches[1], $assoc = TRUE);
 

-    // The "qe" object contains elements which will result in invalid XML
-    // element names, so remove it.
+    // Remove items that may contain elements which can result in invalid XML
+    // element names (for example names starting with a number).

     unset($data["qe"]);
     unset($data["qe"]);

-
-    // The "knobs" object contains elements with undefined namespaces, so
-    // remove it to silence an error message.
-    unset($data["knobs"]);
+    unset($data["knobx"]);
+    unset($data["to_cache"]);
+
+    // Stop here in case Instagram redirected to the login page, this can
+    // happen when too many consecutive requests have been made from the same
+    // IP.
+    if (array_key_exists("LoginAndSignupPage", $data["entry_data"])) {
+      trigger_error("Cannot open instagram page: redirected to Login page.\n", E_USER_WARNING);
+      return NULL;
+    }

 
     $json = json_encode($data);
 
 
     $json = json_encode($data);
 


@@ -335,7 +352,10 @@ class Tweeper {
     // If there is a host-specific method to get the XML data, use it!
     $get_xml_host_method = 'getXml' . Tweeper::toUpperCamelCase($host, '.');
     if (method_exists($this, $get_xml_host_method)) {
     // If there is a host-specific method to get the XML data, use it!
     $get_xml_host_method = 'getXml' . Tweeper::toUpperCamelCase($host, '.');
     if (method_exists($this, $get_xml_host_method)) {

-      $xml_data = call_user_func_array(array($this, $get_xml_host_method), array($html));
+      $xml_data = call_user_func_array([$this, $get_xml_host_method], [$html]);
+      if (NULL === $xml_data) {
+        return NULL;
+      }

       $xmlDoc->loadXML($xml_data);
     }
     else {
       $xmlDoc->loadXML($xml_data);
     }
     else {


@@ -363,7 +383,7 @@ class Tweeper {
       return NULL;
     }
 
       return NULL;
     }
 

-    $stylesheet_contents = Tweeper::getUrlContents($stylesheet);
+    $stylesheet_contents = file_get_contents($stylesheet);

     if (FALSE === $stylesheet_contents) {
       trigger_error("Cannot open $stylesheet", E_USER_WARNING);
       return NULL;
     if (FALSE === $stylesheet_contents) {
       trigger_error("Cannot open $stylesheet", E_USER_WARNING);
       return NULL;


@@ -394,7 +414,7 @@ class Tweeper {
 
     if (TRUE === $validate_scheme) {
       $scheme = $url["scheme"];
 
     if (TRUE === $validate_scheme) {
       $scheme = $url["scheme"];

-      if (!in_array($scheme, array("http", "https"))) {
+      if (!in_array($scheme, ["http", "https"])) {

         trigger_error("unsupported scheme: $scheme", E_USER_WARNING);
         return NULL;
       }
         trigger_error("unsupported scheme: $scheme", E_USER_WARNING);
         return NULL;
       }


@@ -423,7 +443,7 @@ class Tweeper {
 
     $preprocess_html_host_method = 'preprocessHtml' . Tweeper::toUpperCamelCase($host, '.');
     if (method_exists($this, $preprocess_html_host_method)) {
 
     $preprocess_html_host_method = 'preprocessHtml' . Tweeper::toUpperCamelCase($host, '.');
     if (method_exists($this, $preprocess_html_host_method)) {

-      $html = call_user_func_array(array($this, $preprocess_html_host_method), array($html));
+      $html = call_user_func_array([$this, $preprocess_html_host_method], [$html]);

     }
 
     $xmlDoc = $this->htmlToXml($html, $host);
     }
 
     $xmlDoc = $this->htmlToXml($html, $host);