Nextcloud-News/lib/Fetcher/FeedFetcher.php

<?php
/**
 * Nextcloud - News
 *
 * This file is licensed under the Affero General Public License version 3 or
 * later. See the COPYING file.
 *
 * @author    Alessandro Cosentino <cosenal@gmail.com>
 * @author    Bernhard Posselt <dev@bernhard-posselt.com>
 * @copyright 2012 Alessandro Cosentino
 * @copyright 2012-2014 Bernhard Posselt
 */

namespace OCA\News\Fetcher;

use DateTime;
use Favicon\Favicon;
use FeedIo\Feed\ItemInterface;
use FeedIo\FeedInterface;
use FeedIo\FeedIo;
use FeedIo\Reader\ReadErrorException;
use GuzzleHttp\Client;
use GuzzleHttp\Exception\RequestException;
use GuzzleHttp\Exception\ConnectException;

use Net_URL2;
use OCP\IL10N;
use OCP\ITempManager;

use OCA\News\Db\Item;
use OCA\News\Db\Feed;
use OCA\News\Utility\Time;
use OCA\News\Scraper\Scraper;
use OCA\News\Config\FetcherConfig;
use Psr\Log\LoggerInterface;
use SimpleXMLElement;

class FeedFetcher implements IFeedFetcher
{

    /**
     * @var Favicon
     */
    private $faviconFactory;

    /**
     * @var FeedIo
     */
    private $reader;

    /**
     * @var Scraper
     */
    private $scraper;

    /**
     * @var IL10N
     */
    private $l10n;

    /**
     * @var ITempManager
     */
    private $ITempManager;

    /**
     * @var Time
     */
    private $time;

    /**
     * @var LoggerInterface
     */
    private $logger;

    public function __construct(
        FeedIo $fetcher,
        Favicon $favicon,
        Scraper $scraper,
        IL10N $l10n,
        ITempManager $ITempManager,
        Time $time,
        LoggerInterface $logger
    ) {
        $this->reader         = $fetcher;
        $this->faviconFactory = $favicon;
        $this->scraper        = $scraper;
        $this->l10n           = $l10n;
        $this->ITempManager   = $ITempManager;
        $this->time           = $time;
        $this->logger         = $logger;
    }


    /**
     * This fetcher handles all the remaining urls therefore always returns true.
     *
     * @param string $url The URL to check
     *
     * @return bool
     */
    public function canHandle(string $url): bool
    {
        return true;
    }


    /**
     * Fetch a feed from remote
     *
     * @inheritdoc
     */
    public function fetch(
        string $url,
        bool $fullTextEnabled,
        ?string $user,
        ?string $password,
        ?string $httpLastModified
    ): array {
        $url2 = new Net_URL2($url);
        if (!is_null($user) && trim($user) !== '') {
            $url2->setUserinfo(rawurlencode($user), rawurlencode($password));
        }
        if (!is_null($httpLastModified) && trim($httpLastModified) !== '') {
            $lastModified = new DateTime($httpLastModified);
        } else {
            $lastModified = null;
        }
        $url = $url2->getNormalizedURL();
        $this->reader->resetFilters();
        $resource = $this->reader->read($url, null, $lastModified);

        $location     = $resource->getUrl();
        $parsedFeed   = $resource->getFeed();
        $feed = $this->buildFeed(
            $parsedFeed,
            $url,
            $location
        );

        $items = [];
        $RTL = $this->determineRtl($parsedFeed);
        $feedName = $parsedFeed->getTitle();
        $feedAuthor = $parsedFeed->getAuthor();
        $this->logger->debug(
            'Feed {url} was modified since last fetch. #{count} items',
            [
            'url'   => $url,
            'count' => count($parsedFeed),
            ]
        );

        foreach ($parsedFeed as $item) {
            $body = null;
            $currRTL = $RTL;

            // Scrape the content if full-text is enabled and if the feed provides a URL
            if ($fullTextEnabled) {
                $itemLink = $item->getLink();
                if ($itemLink !== null && $this->scraper->scrape($itemLink)) {
                    $body = $this->scraper->getContent();
                    $currRTL = $this->scraper->getRTL($currRTL);
                }
            }

            $builtItem = $this->buildItem($item, $body, $currRTL, $feedAuthor);
            $this->logger->debug(
                'Added item {title} for feed {feed} lastmodified: {datetime}',
                [
                'title' => $builtItem->getTitle(),
                'feed'  => $feedName,
                'datetime'  => $builtItem->getLastModified(),
                ]
            );
            $items[] = $builtItem;
        }

        return [$feed, $items];
    }

    /**
     * Decode the string twice
     *
     * @param string $string String to decode
     *
     * @return string
     */
    private function decodeTwice(string $string): string
    {
        return html_entity_decode(
            html_entity_decode(
                $string,
                ENT_QUOTES | ENT_HTML5,
                'UTF-8'
            ),
            ENT_QUOTES | ENT_HTML5,
            'UTF-8'
        );
    }

    /**
     * Check if a feed is RTL or not
     *
     * @param FeedInterface $parsedFeed The feed that was parsed
     *
     * @return bool
     */
    protected function determineRtl(FeedInterface $parsedFeed): bool
    {
        $language = $parsedFeed->getLanguage();

        $language = strtolower($language);
        $rtl_languages = array(
            'ar', // Arabic (ar-**)
            'fa', // Farsi (fa-**)
            'ur', // Urdu (ur-**)
            'ps', // Pashtu (ps-**)
            'syr', // Syriac (syr-**)
            'dv', // Divehi (dv-**)
            'he', // Hebrew (he-**)
            'yi', // Yiddish (yi-**)
        );
        foreach ($rtl_languages as $prefix) {
            if (strpos($language, $prefix) === 0) {
                return true;
            }
        }
        return false;
    }

    /**
     * Build an item based on a feed.
     *
     * @param ItemInterface $parsedItem The item to use
     * @param string|null   $body       Text of the item, if not provided use description from $parsedItem
     * @param bool          $RTL        True if the feed is RTL (Right-to-left)
     * @param string|null   $feedAuthor Author of the feed as fallback when the item has no Author
     *
     * @return Item
     */
    protected function buildItem(
        ItemInterface $parsedItem,
        ?string $body = null,
        bool $RTL = false,
        $feedAuthor = null
    ): Item {
        $item = new Item();
        $item->setUnread(true);
        $itemLink = $parsedItem->getLink();
        $itemTitle = $parsedItem->getTitle();
        $item->setUrl($itemLink);
        $publicId = $parsedItem->getPublicId();
        if ($publicId == null) {
            // Fallback on using the URL as the guid for the feed item if no guid provided by feed
            $this->logger->debug(
                "Feed item {title} with link {link} did not expose a guid, falling back to using link as guid",
                [
                'title' => $itemTitle,
                'link' => $itemLink
                ]
            );
            $publicId = $itemLink;
        }
        if ($publicId == null) {
            throw new ReadErrorException("Malformed feed: item has no GUID");
        }
        $item->setGuid($publicId);
        $item->setGuidHash(md5($item->getGuid()));

        $lastModified = $parsedItem->getLastModified() ?? new DateTime();
        if ($parsedItem->getValue('pubDate') !== null) {
            $pubDT = new DateTime($parsedItem->getValue('pubDate'));
        } elseif ($parsedItem->getValue('published') !== null) {
            $pubDT = new DateTime($parsedItem->getValue('published'));
        } else {
            $pubDT = $lastModified;
        }

        $item->setPubDate($pubDT->getTimestamp());

        $item->setLastModified($lastModified->getTimestamp());
        $item->setRtl($RTL);

        // unescape content because angularjs helps against XSS
        if ($itemTitle !== null) {
            $item->setTitle($this->decodeTwice($itemTitle));
        }

        $author = $parsedItem->getAuthor() ?? $feedAuthor;

        if ($author !== null && $author->getName() !== null) {
            $item->setAuthor($this->decodeTwice($author->getName()));
        }

        $categories = [];
        foreach ($parsedItem->getCategories() as $category) {
            if ($category->getLabel() !== null) {
                $categories[] = $this->decodeTwice($category->getLabel());
            }
        }
        $item->setCategories($categories);

        // Use description from feed if body is not provided (by a scraper)
        if ($body === null) {
            $body = $parsedItem->getValue('content:encoded')
                    ?? $parsedItem->getContent()
                    ?? $parsedItem->getSummary();
        }

        // purification is done in the service layer
        if (!is_null($body)) {
            $body = mb_convert_encoding(
                $body,
                'HTML-ENTITIES',
                mb_detect_encoding($body)
            );
            if (strpos($body, 'CDATA') !== false) {
                libxml_use_internal_errors(true);
                $data = simplexml_load_string(
                    "<?xml version=\"1.0\"?><item>$body</item>",
                    SimpleXMLElement::class,
                    LIBXML_NOCDATA
                );
                if ($data !== false && libxml_get_last_error() === false) {
                    $body = (string) $data;
                }
                libxml_clear_errors();
            }
        }

        $item->setBody($body);

        if ($parsedItem->hasMedia()) {
            // TODO: Fix multiple media support
            foreach ($parsedItem->getMedias() as $media) {
                if (!$item->isSupportedMime($media->getType())
                    && !$media->getThumbnail()
                    && !$media->getDescription()
                ) {
                    continue;
                }
                $item->setEnclosureMime($media->getType());
                $item->setEnclosureLink($media->getUrl());
                $item->setMediaThumbnail($media->getThumbnail());
                if ($media->getDescription()) {
                    $description = str_replace("\n", "<br>", $media->getDescription());
                    $item->setMediaDescription($description);
                }
            }
        }

        $item->generateSearchIndex();
        return $item;
    }

    /**
     * Return the favicon for a given feed and url
     *
     * @param FeedInterface $feed Feed to check for a logo
     * @param string        $url  Original URL for the feed
     *
     * @return string|null
     */
    protected function getFavicon(FeedInterface $feed, string $url): ?string
    {
        $favicon = null;
        // trim the string because authors do funny things
        $feed_logo = $feed->getLogo();
        
        if (!is_null($feed_logo)) {
            $favicon = trim($feed_logo);
        }
        
        ini_set('user_agent', FetcherConfig::DEFAULT_USER_AGENT);

        $base_url = new Net_URL2($url);
        $base_url->setPath("");
        $base_url = $base_url->getNormalizedURL();

        // Return if the URL is empty
        if ($base_url === null || trim($base_url) === '') {
            return null;
        }

        // check if feed has a logo entry
        if ($favicon === null || $favicon === '') {
            $return = $this->faviconFactory->get($base_url);
            return is_string($return) ? $return : null;
        }

        // logo will be saved in the tmp folder provided by Nextcloud, file is named as md5 of the url
        $favicon_path = join(DIRECTORY_SEPARATOR, [$this->ITempManager->getTempBaseDir(), md5($favicon)]);
        $downloaded = false;

        if (file_exists($favicon_path)) {
            $last_modified = filemtime($favicon_path);
        } else {
            $last_modified = 0;
        }

        try {
            // Base_uri can only be set on creation, will be used when link is relative.
            $client = new Client(['base_uri' => $base_url]);
            $response = $client->request(
                'GET',
                $favicon,
                [
                    'sink' => $favicon_path,
                    'headers' => [
                        'User-Agent'        => FetcherConfig::DEFAULT_USER_AGENT,
                        'Accept'            => 'image/*',
                        'If-Modified-Since' => date(DateTime::RFC7231, $last_modified)
                    ]
                ]
            );
            $downloaded = true;

            $this->logger->debug(
                "Feed:{url} Logo:{logo} Status:{status}",
                [
                'status' => $response->getStatusCode(),
                'url'    => $favicon_path,
                'logo'   => $favicon
                ]
            );
        } catch (RequestException | ConnectException $e) {
            $this->logger->info(
                'An error occurred while trying to download the feed logo of {url}: {error}',
                [
                'url'   => $url,
                'error' => $e->getMessage() ?? 'Unknown'
                ]
            );
        }

        $is_image = $downloaded && substr(mime_content_type($favicon_path), 0, 5) === "image";

        // check if file is actually an image
        if (!$is_image) {
            $return = $this->faviconFactory->get($base_url);
            return is_string($return) ? $return : null;
        }

        list($width, $height, $type, $attr) = getimagesize($favicon_path);
        // check if image is square else fall back to favicon
        if ($width !== $height) {
            $return = $this->faviconFactory->get($base_url);
            return is_string($return) ? $return : null;
        }

        return is_string($favicon) ? $favicon : null;
    }

    /**
     * Build a feed based on provided info
     *
     * @param FeedInterface $feed     Feed to build from
     * @param string        $url      URL to use
     * @param string        $location String base URL
     *
     * @return Feed
     */
    protected function buildFeed(FeedInterface $feed, string $url, string $location): Feed
    {
        $newFeed = new Feed();

        // unescape content because angularjs helps against XSS
        if ($feed->getTitle() !== null) {
            $title = strip_tags($this->decodeTwice($feed->getTitle()));
            $newFeed->setTitle($title);
        }
        $newFeed->setUrl($url);  // the url used to add the feed
        $newFeed->setLocation($location);  // the url where the feed was found
        $newFeed->setLink($feed->getLink());  // <link> attribute in the feed
        if ($feed->getLastModified() instanceof DateTime) {
            $newFeed->setHttpLastModified($feed->getLastModified()->format(DateTime::RSS));
        }
        $newFeed->setAdded($this->time->getTime());



        $favicon = $this->getFavicon($feed, $url);
        $newFeed->setFaviconLink($favicon);

        return $newFeed;
    }
}