<?php
/**
* SeekQuarry/Yioop --
* Open Source Pure PHP Search Engine, Crawler, and Indexer
*
* Copyright (C) 2009 - 2026 Chris Pollett chris@pollett.org
*
* LICENSE:
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* END LICENSE
*
* @author Chris Pollett chris@pollett.org (initial MediaJob class
* and subclasses based on work of Pooja Mishra for her master's)
* @license https://www.gnu.org/licenses/ GPL3
* @link https://www.seekquarry.com/
* @copyright 2009 - 2026
* @filesource
*/
namespace seekquarry\yioop\library\media_jobs;
use seekquarry\yioop\configs as C;
use seekquarry\yioop\library as L;
use seekquarry\yioop\library\CrawlConstants;
use seekquarry\yioop\library\FetchUrl;
use seekquarry\yioop\library\storage_formats\IndexShard;
use seekquarry\yioop\library\language_processing\PhraseParser;
use seekquarry\yioop\library\UrlParser;
use seekquarry\yioop\models\GroupModel;
use seekquarry\yioop\models\WikiModel;
use seekquarry\yioop\controllers\CrawlController;
/**
* A media job to periodically download Podcasts and store them as resources
* of a Wiki Page
*/
class PodcastDownloadJob extends MediaJob
{
/**
* FFMPEG_LINES_LOGGED is how many of ffmpeg's last lines are written
* to the log when a fetch goes wrong. Enough to say why without
* filling the log with what it says while working.
*/
const FFMPEG_LINES_LOGGED = 20;
/**
* SECONDS_BETWEEN_SAYINGS is how long is left between two lines of a
* running command's progress being passed along. ffmpeg says where
* it has got to several times a second, and a watcher needs only to
* see that it is still moving.
*/
const SECONDS_BETWEEN_SAYINGS = 5;
/**
* BYTES_READ_AT_ONCE is how much of a running command's output is
* taken in one read. Small enough that a line of progress is seen
* soon after it is written.
*/
const BYTES_READ_AT_ONCE = 256;
/**
* how long in seconds before a podcast item expires
*/
const ITEM_EXPIRES_TIME = C\ONE_WEEK;
/**
* Mamimum number of feeds to download in one try
*/
const MAX_PODCASTS_ONE_GO = 100;
/**
* Time in current epoch when feeds last updated
* @var int
*/
public $update_time;
/**
* Whether the current run was triggered by an immediate-update
* request rather than the periodic interval, in which case only
* the requested folders' sources are downloaded
* @var bool
*/
public $request_driven = false;
/**
* Folder keys with a pending immediate-update request for the
* current run, set by checkPrerequisites
* @var array
*/
public $requested_folder_keys = [];
/**
* Instance of group model used to get a list of podcasts
* @var object
*/
public $group_model;
/**
* $wiki_model is the instance of the wiki model this job reads and
* writes group wiki pages and their resource folders through: where
* a podcast's target page lives, which folders hold its downloads,
* and the copying in and deleting of downloaded items. init() sets
* it from the controller where the job runs inside the web app, and
* builds one itself where the job runs under the media updater,
* which has no controller.
* @var object
*/
public $wiki_model;
/**
* Datasource object used to run db queries related to fes items
* (for storing and updating them)
* @var object
*/
public $db;
/**
* Initializes the last update time to far in the past so, feeds will get
* immediately updated. Sets up connect to DB to store feeds items, and
* makes it so the same media job runs both on name server and client
* Media Updaters
*/
public function init()
{
$this->update_time = 0;
$this->name_server_does_client_tasks = true;
$this->name_server_does_client_tasks_only = true;
$db_class = C\NS_DATASOURCES . ucfirst(C\p('DBMS')). "Manager";
$this->db = new $db_class();
$this->db->connect();
if(!empty($this->controller)) {
$group_model = $this->controller->model("group");
$wiki_model = $this->controller->model("wiki");
} else {
$group_model = new GroupModel();
$wiki_model = new WikiModel();
}
$this->group_model = $group_model;
$this->wiki_model = $wiki_model;
C\nsconddefine("PODCAST_UPDATE_INTERVAL", C\ONE_DAY);
}
/**
* Builds the per-folder key that identifies one podcast
* destination across the download marker and the update-request
* marker. A destination is a wiki resource folder, fixed by its
* group, page, and sub-path, so the key combines those three.
*
* @param int $group_id group the destination page belongs to
* @param int $page_id wiki page the podcast downloads to
* @param string $sub_path resource sub-folder within the page,
* empty for the page's top resource folder
* @return string key naming this destination folder
*/
public static function podcastFolderKey($group_id, $page_id,
$sub_path)
{
$clean_sub_path = ($sub_path === "" || $sub_path === null) ?
"" : L\crawlHash($sub_path);
return "Podcast-" . $group_id . "-" . $page_id . "-" .
$clean_sub_path;
}
/**
* Path of the marker file the job writes and refreshes while it
* is downloading podcast items to one destination folder. The
* file's modification time is the heartbeat the Media List page
* checks against PODCAST_DOWNLOAD_STALE_TIME.
*
* @param string $folder_key key from podcastFolderKey
* @return string absolute path of the downloading marker file
*/
public static function podcastDownloadingMarkerFile($folder_key)
{
return C\SCHEDULES_DIR . "/" . $folder_key . "Downloading.txt";
}
/**
* Path of the marker file the web UI writes to request an
* immediate update of one destination folder's podcast sources.
* The job removes it once it has begun honoring the request.
*
* @param string $folder_key key from podcastFolderKey
* @return string absolute path of the update-request marker file
*/
public static function podcastRequestMarkerFile($folder_key)
{
return C\SCHEDULES_DIR . "/" . $folder_key . "Request.txt";
}
/**
* Whether a destination folder is currently being downloaded to,
* judged by a downloading marker whose heartbeat is newer than
* PODCAST_DOWNLOAD_STALE_TIME. A marker left behind by an updater
* that died mid-download goes stale and reads as not downloading.
*
* @param string $folder_key key from podcastFolderKey
* @return bool true if a fresh downloading marker exists
*/
public static function isPodcastDownloading($folder_key)
{
$marker = self::podcastDownloadingMarkerFile($folder_key);
clearstatcache(true, $marker);
if (!file_exists($marker)) {
return false;
}
return (time() - filemtime($marker)) <
C\PODCAST_DOWNLOAD_STALE_TIME;
}
/**
* Writes an immediate-update request for a destination folder so
* the next MediaUpdater loop downloads that folder's podcast
* sources without waiting out the periodic interval. Refuses to
* write one while the folder is already downloading, so a button
* press during an active download cannot start a second one.
*
* @param string $folder_key key from podcastFolderKey
* @return bool true if the request was written, false if a
* download is already in progress for the folder
*/
public static function requestPodcastUpdate($folder_key)
{
if (self::isPodcastDownloading($folder_key)) {
return false;
}
file_put_contents(
self::podcastRequestMarkerFile($folder_key), time());
return true;
}
/**
* Refreshes the heartbeat on a folder's downloading marker so a
* download that runs longer than PODCAST_DOWNLOAD_STALE_TIME
* still reads as active. Does nothing when the marker is absent
* (the download already finished or was never started for this
* folder), so it is safe to call from the per-item loop.
*
* @param string $folder_key key from podcastFolderKey
*/
public static function heartbeatPodcastDownload($folder_key)
{
if ($folder_key === "" || $folder_key === null) {
return;
}
$marker = self::podcastDownloadingMarkerFile($folder_key);
if (file_exists($marker)) {
touch($marker);
}
}
/**
* Only update if its been more than an hour since the last update
*
* @return bool whether its been an hour since the last update
*/
public function checkPrerequisites()
{
$time = time();
$delta = $time - $this->update_time;
$this->requested_folder_keys = self::pendingPodcastRequests();
if (!empty($this->requested_folder_keys)) {
$this->request_driven = true;
L\crawlLog("Performing requested podcast folder update");
return true;
}
if ($delta > C\PODCAST_UPDATE_INTERVAL) {
$this->update_time = $time;
$this->request_driven = false;
L\crawlLog("Performing media podcasts update");
return true;
}
L\crawlLog("Time since last update not exceeded, skipping wiki
update");
return false;
}
/**
* Folder keys with a pending immediate-update request, taken from
* the request marker files in the schedules directory. A run
* triggered by a request downloads only these folders' sources.
*
* @return array list of folder-key strings with a pending request
*/
public static function pendingPodcastRequests()
{
$keys = [];
$pattern = C\SCHEDULES_DIR . "/Podcast-*Request.txt";
clearstatcache();
foreach (glob($pattern) as $path) {
$name = basename($path);
$keys[] = substr($name, 0, -strlen("Request.txt"));
}
return $keys;
}
/**
* Get the media sources from the local database and use those to run the
* the same task as in the distributed setting
*/
public function nondistributedTasks()
{
$db = $this->db;
$sql = "SELECT * FROM MEDIA_SOURCE WHERE (TYPE='feed_podcast' OR ".
"TYPE='scrape_podcast')";
$result = $db->execute($sql);
$podcasts = [];
if ($this->request_driven) {
L\crawlLog("----Folders asked for by a forced update: " .
implode(", ", $this->requested_folder_keys));
}
while ($podcast = $db->fetchArray($result)) {
$this->parsePodcastAuxInfo($podcast);
/* Both of the ways a podcast is passed over say so. A run
that ends up responsible for nothing used to give no
reason at all, so a forced update that downloaded nothing
looked the same as one that had nothing to do. */
if (!isset($podcast["PREVIOUSLY_DOWNLOADED"])) {
L\crawlLog("----Skipping {$podcast['NAME']}: its wiki " .
"page folder could not be read; page is " .
"'{$podcast['WIKI_PAGE']}'");
continue;
}
if ($this->request_driven && !in_array(
$podcast["FOLDER_KEY"], $this->requested_folder_keys)) {
L\crawlLog("----Skipping {$podcast['NAME']}: it fills " .
"folder {$podcast['FOLDER_KEY']}, which no forced " .
"update asked for");
continue;
}
$podcasts[] = $podcast;
}
if ($this->request_driven) {
/* A folder asked for that no podcast source fills would
otherwise leave the run silent, since the loop above only
reports the sources it passed over. */
$filled = [];
foreach ($podcasts as $podcast) {
$filled[$podcast["FOLDER_KEY"]] = true;
}
foreach ($this->requested_folder_keys as $folder_key) {
if (empty($filled[$folder_key])) {
L\crawlLog("----A forced update asked for folder " .
"$folder_key, which no podcast source fills");
}
}
foreach ($this->requested_folder_keys as $folder_key) {
$request_marker =
self::podcastRequestMarkerFile($folder_key);
if (file_exists($request_marker)) {
unlink($request_marker);
}
}
}
$this->tasks = $podcasts;
$this->doTasks($podcasts);
}
/**
* For each podcast source downloads the podcast web file, checks which
* podcast items are not in the database, adds them.
*
* @param array $tasks array of feed info (url to download, paths to
* extract etc)
*/
public function doTasks($tasks)
{
if (!is_array($tasks)) {
L\crawlLog(
"----This media updater is NOT responsible for any podcasts!");
return;
}
$podcasts = $tasks;
L\crawlLog("----This media updater is responsible for the podcasts:");
$i = 1;
foreach ($podcasts as $podcast) {
L\crawlLog("---- $i. " . $podcast["NAME"]);
$i++;
}
$num_podcasts = count($podcasts);
$podcasts_one_go = self::MAX_PODCASTS_ONE_GO;
$limit = 0;
while ($limit < $num_podcasts) {
$podcasts_batch = array_slice($podcasts, $limit,
$podcasts_one_go);
$this->updatePodcastsOneGo($podcasts_batch);
$limit += $podcasts_one_go;
}
}
/**
* Used to fill in details for an associative arrays containing the
* details of a Wiki feed and scrape podcast which should be examined
* to see if new items should be downloaded to wiki pages. As part of
* processing expired feed items for the given wiki might be deleted.
*
* @param array &$podcast after running will contain an associative
* array of details about a particular podcast. The input podcast
* is assumed to have at least the NAME, WIKI_PAGE, AUX_PATH, and CATEGORY
* fields filled in. The latter with the time in seconds till item
* expires. If successful the MAX_AGE (which is esseentially the value
* the CATEGORY field), WIKI_FILE_PATTERN, WIKI_PAGE_FOLDERS, and
* PREVIOUSLY_DOWNLOADED folders will be filled in.
* @param boolean $test_mode if true then does not cull expired feed items
* from disk, but will return previously downloaded as if it had.
*/
public function parsePodcastAuxInfo(&$podcast, $test_mode = false)
{
$wiki_model = $this->wiki_model;
$locale_tag = $podcast["LANGUAGE"];
$aux_parts = explode("###",
html_entity_decode($podcast['AUX_INFO'], ENT_QUOTES));
list($podcast['AUX_URL_XPATH'], , , , $podcast['LINK_XPATH'],
$podcast['WIKI_PAGE']) = $aux_parts;
$podcast['MAX_AGE'] = (empty($podcast['CATEGORY'])) ?
self::ITEM_EXPIRES_TIME : $podcast['CATEGORY'];
$group_model = $this->group_model;
list($group_id, $page_id, $sub_path, $podcast["WIKI_FILE_PATTERN"]) =
$wiki_model->getGroupIdPageIdSubPathFromName($podcast['WIKI_PAGE'],
$locale_tag);
$podcast["GROUP_ID"] = $group_id;
$podcast["PAGE_ID"] = $page_id;
$podcast["SUB_PATH"] = $sub_path;
$podcast["FOLDER_KEY"] = self::podcastFolderKey($group_id,
$page_id, $sub_path);
$podcast["WIKI_PAGE_FOLDERS"] =
$wiki_model->getGroupPageResourcesFolders($group_id, $page_id,
$sub_path, true);
if (empty($podcast["WIKI_PAGE_FOLDERS"][1])) {
return;
}
$podcast["PREVIOUSLY_DOWNLOADED"] = [];
list($resource_folder, $thumb_folder,) = $podcast["WIKI_PAGE_FOLDERS"];
$podcast_download_file = $thumb_folder . "/" .
L\crawlHash($podcast["NAME"]) . ".txt";
if (file_exists($podcast_download_file)) {
$podcast["PREVIOUSLY_DOWNLOADED"] = unserialize(file_get_contents(
$podcast_download_file));
$previously_downloaded = [];
$max_age = $podcast["MAX_AGE"];
$time = time();
foreach ($podcast["PREVIOUSLY_DOWNLOADED"] as $guid => $item) {
if ($max_age < 0||$item["SAVE_TIMESTAMP"] + $max_age > $time) {
$previously_downloaded[$guid] = $item;
} else if (!$test_mode) {
$wiki_model->deleteResource($item['FILENAME'], $group_id,
$page_id, $sub_path);
}
}
$podcast["PREVIOUSLY_DOWNLOADED"] = $previously_downloaded;
}
}
/**
* For each of a supplied list of podcast associative arrays,
* downloads the non-expired media for that podcast to the wiki folder
* specified.
*
* @param array &$podcasts an array of associative arrays of info about
* how and where to download podcasts to
* @param int $age oldest age items to consider for download
* @param boolean $test_mode if true then rather then updating items in
* wiki, returns as a string summarizing the results of the downloads
* that would occur as part of updating the podcast
* @return mixed either true, or if $test_mode is true then the results
* as a string of the operations involved in downloading the podcasts
*/
public function updatePodcastsOneGo($podcasts, $age = C\ONE_WEEK,
$test_mode = false)
{
$test_results = "";
$log_function = function ($msg, $log_tag = "div class='source-test'")
use (&$test_results, $test_mode) {
$close_tag= preg_split("/\s+/",$log_tag)[0];
if ($test_mode) {
$test_results .= "<$log_tag>$msg</$close_tag>\n";
} else {
L\crawlLog($msg);
}
};
$podcasts = FetchUrl::getPages($podcasts, false, C\ONE_GIB,
null, "SOURCE_URL", CrawlConstants::PAGE, false, null, true);
foreach ($podcasts as $podcast) {
if (empty($podcast[CrawlConstants::PAGE])) {
$log_function(
"...No data in {$podcast['NAME']} feed skipping.", "h3");
continue;
}
$log_function("----Updating {$podcast['NAME']}.", "h3");
/* strip namespaces */
$page = preg_replace('@<(/?)(\w+\s*)\:@u', '<$1',
$podcast[CrawlConstants::PAGE]);
if (!empty($page)) {
$podcast[CrawlConstants::PAGE] = $page;
}
$mime_type = (empty($podcast[CrawlConstants::TYPE])) ?
"application/xml" : $podcast[CrawlConstants::TYPE];
$is_html = preg_match("/html/i", $mime_type);
if ($test_mode) {
$log_function("...Downloaded Podcast Data was:",
"h3");
$out_dom = new \DOMDocument('1.0');
$out_dom->preserveWhiteSpace = false;
$out_dom->formatOutput = true;
if ($is_html) {
set_error_handler(null);
@$out_dom->loadHTML($podcast[CrawlConstants::PAGE]);
restore_error_handler();
$podcast_contents = $out_dom->saveHTML();
} else {
set_error_handler(null);
@$out_dom->loadXML($podcast[CrawlConstants::PAGE]);
restore_error_handler();
$podcast_contents = $out_dom->saveXML();
}
$podcast_contents = "<textarea disabled='disabled'>" .
htmlentities($podcast_contents) . "</textarea>";
$log_function($podcast_contents);
}
$marker = "";
if (!$test_mode && !empty($podcast["FOLDER_KEY"])) {
$marker = self::podcastDownloadingMarkerFile(
$podcast["FOLDER_KEY"]);
file_put_contents($marker, time());
}
if ($is_html && !empty($podcast['LINK_XPATH'])) {
list($num_added, $test_info) =
$this->processHtmlPodcast($podcast, $age, $test_mode);
} else if (!$is_html) {
list($num_added, $test_info) =
$this->processFeedPodcast($podcast, $age, $test_mode);
} else {
$num_added = 0;
$test_info = "";
}
if ($marker !== "" && file_exists($marker)) {
unlink($marker);
}
$test_results .= $test_info;
if ($num_added > 0 && !$test_mode) {
$podcast_download_file = $podcast["WIKI_PAGE_FOLDERS"][1] .
"/" . L\crawlHash($podcast["NAME"]) . ".txt";
file_put_contents($podcast_download_file,
serialize($podcast["PREVIOUSLY_DOWNLOADED"]));
} else if ($num_added <= 0) {
$log_function("----could not parse any podcasts from page.");
}
}
return ($test_mode) ? $test_results : true;
}
/**
* Used to download the media item associated with an HTML scrape podcast
*
* @param array &$podcast associative array containing info about the
* location, how to handle, and where to download the podcast
* @param int $age max age of an the media item to be considered for
* download
* @param boolean $test_mode if true then rather then updating items in
* wiki, returns as a string summarizing the results of the downloads
* that would occur as part of updating the podcast
* @return array [whether item downloaded, test_mode_info_string if
* applicable or "" otherwise]
*/
public function processHtmlPodcast(&$podcast, $age, $test_mode = false)
{
$test_results = "";
$log_function = function ($msg, $log_tag = "div class='source-test'")
use (&$test_results, $test_mode) {
$close_tag= preg_split("/\s+/",$log_tag)[0];
if ($test_mode) {
$test_results .= "<$log_tag>$msg</$close_tag>\n";
} else {
L\crawlLog($msg);
}
};
$page = $podcast[CrawlConstants::PAGE];
$dom = L\getDomFromString($page);
$source_url = $podcast["SOURCE_URL"];
if (!empty($podcast['AUX_URL_XPATH'])) {
$sub_aux_xpaths = explode("\n", $podcast['AUX_URL_XPATH']);
$log_function("...Processing the following AUX PATHS:", "h3");
$log_function(print_r($sub_aux_xpaths, true));
foreach ($sub_aux_xpaths as $aux_xpath) {
$aux_url = $this->getLinkFromQueryPage($aux_xpath,
$page, $dom, $source_url);
if (empty($aux_url)) {
$log_function("Downloading aux_url for xpath $aux_xpath ".
"was empty, bailing...", "h3");
break;
}
$log_function("Downloading aux_url $aux_url", "h3");
$page = FetchUrl::getPage($aux_url);
if ($test_mode) {
$log_function("...Downloaded Aux Data was:",
"h3");
$out_dom = new \DOMDocument('1.0');
$out_dom->preserveWhiteSpace = false;
$out_dom->formatOutput = true;
set_error_handler(null);
@$out_dom->loadHTML($page);
restore_error_handler();
$aux_contents = $out_dom->saveHTML();
$aux_contents = "<textarea disabled='disabled'>" .
htmlentities($aux_contents) . "</textarea>";
$log_function($aux_contents);
}
$dom = L\getDomFromString($page);
}
}
$url = $this->getLinkFromQueryPage($podcast['LINK_XPATH'], $page, $dom,
$source_url);
if (empty($aux_url)) {
$aux_url = $url;
}
$log_function("----...done extracting info. Check for new ".
"podcast item in {$podcast['NAME']}.", "h3");
if ($test_mode) {
$log_function("----...Found following item...", "h3");
}
$item = ['guid' => L\crawlHash($url), 'link' => $url,
'pubdate' => time(), 'title' => $aux_url];
if ($test_mode) {
$log_function(print_r($item, true));
$did_add = true;
} else {
$did_add = $this->downloadPodcastItemIfNew($item, $podcast,
$age);
}
if ($did_add) {
return [1, $test_results];
}
}
/**
* Used to extract a URL from a page either as a string of in dom form
* and to canonicalize it based on a starting url.
*
* @param string $xpath either an xpath to look into a dom object or
* a regex to search a page as a string
* @param string $page source page to search in as a string
* @param string $dom source page as a dom object
* @param string $source_url url to use to canonicalize an incomplete
* url if the extraction only produces part of a url
* @return string desired url link
*/
public function getLinkFromQueryPage($xpath, $page, $dom, $source_url)
{
$dom_xpath = new \DOMXPath($dom);
set_error_handler(null);
$nodes = @$dom_xpath->evaluate($xpath);
restore_error_handler();
$url = false;
if ($nodes === false) {
$regex_json_parts = explode("json|", $xpath);
set_error_handler(null);
@preg_match_all(trim($regex_json_parts[0]), $page, $matches);
restore_error_handler();
if (!empty($matches[1][0])) {
$url = $matches[1][0];
if (!empty($regex_json_parts[1])) {
$json_data = json_decode(trim($url, ";"), true);
$play_list_path = explode("|", $regex_json_parts[1]);
foreach ($play_list_path as $path) {
if (!empty($json_data[$path])) {
$json_data = $json_data[$path];
}
}
$url = $json_data;
} else if (isset($regex_json_parts[1]) && $url[0] == "\"") {
$url = json_decode($url);
}
if (is_string($url)) {
$url = preg_replace('/\\\u002F/i', "/", $url);
} else {
$url = "";
}
}
} else if ($nodes && $nodes->item(0)) {
$node_item = $nodes->item(0);
if (in_array($node_item->nodeName, ["a", "meta", "link"])) {
$href = $node_item->attributes->getNamedItem("href");
if(!empty($href)) {
$url = $href->nodeValue;
}
if (!$url && $node_item->nodeName == 'meta') {
$content = $node_item->attributes->getNamedItem("content");
if(!empty($content)) {
$url = $content->nodeValue;
}
}
} else if (in_array($node_item->nodeName, ["video", "audio"])) {
$src = $node_item->attributes->getNamedItem("src");
if(!empty($src)) {
$url = $src->nodeValue;
}
}
if (!$url) {
$url = $nodes->item(0)->nodeValue;
}
}
if ($url) {
$url = UrlParser::canonicalLink(
urldecode(html_entity_decode($url)), $source_url);
}
return $url;
}
/**
* Processes the page contents of one podcast feed. Determines which
* podcast files on that page are fresh and if a podcast is fresh downloads
* it and moves it to the appropariate wiki folder.
*
* @param array &$podcast associative array containing page data
* for a podcast feed page (not the video or audio files of a particular
* podcast on that page) together with rules for how to process it
* @param int $age how many seconds ago is still considered a recent
* enough podcast to process
* @param boolean $test_mode if true then rather then updating items in
* wiki, returns as a string summarizing the results of the downloads
* that would occur as part of updating the podcast
* @return mixed either true, or if $test_mode is true then the results
* as a string of the operations involved in downloading the podcasts
*/
public function processFeedPodcast(&$podcast, $age, $test_mode = false)
{
$test_results = "";
$log_function = function ($msg, $log_tag = "div class='source-test'")
use (&$test_results, $test_mode) {
$close_tag= preg_split("/\s+/",$log_tag)[0];
if ($test_mode) {
$test_results .= "<$log_tag>$msg</$close_tag>\n";
} else {
L\crawlLog($msg);
}
};
$page = $podcast[CrawlConstants::PAGE];
$page = preg_replace("@<link@", "<slink", $page);
$page = preg_replace("@</link@", "</slink", $page);
$page = preg_replace("@pubDate@i", "pubdate", $page);
$page = preg_replace("@<@", "<", $page);
$page = preg_replace("@>@", ">", $page);
$page = preg_replace("@<!\[CDATA\[(.+?)\]\]>@s", '$1', $page);
/* we also need a hack to make UTF-8 work correctly */
$dom = L\getDomFromString($page);
$dom->encoding = 'UTF-8';
$nodes = $dom->getElementsByTagName('item');
/* see above comment on why slink rather than link */
$item_elements = ["title" => "title",
"description" => "description", "link" =>"slink",
"guid" => "guid", "pubdate" => "pubdate"];
if ($nodes->length == 0) {
/* maybe we're dealing with atom rather than rss */
$nodes = $dom->getElementsByTagName('entry');
$item_elements = [
"title" => "title",
"description" => ["summary", "content"],
"link" => "slink", "guid" => "id",
"pubdate" => "updated"];
}
if (!empty($podcast['LINK_XPATH'])) {
$item_elements['link'] = $podcast['LINK_XPATH'];
}
$log_function("----...done extracting info. Check for new ".
"podcast items in {$podcast['NAME']}.", "h3");
if ($test_mode) {
$log_function("----...Found following items...", "h3");
}
$num_added = 0;
$num_seen = 0;
foreach ($nodes as $node) {
$item = [];
foreach ($item_elements as $db_element => $podcast_element) {
if (!$test_mode) {
L\crawlTimeoutLog(
"----still adding podcast items to index.");
}
if ($db_element == "link" && substr($podcast_element, 0, 4)
== "http") {
$element_text = $podcast_element;
$item[$db_element] = strip_tags($element_text);
continue;
}
if (!is_array($podcast_element)) {
$podcast_element = [$podcast_element];
}
foreach ($podcast_element as $tag_name) {
$tag_node = $node->getElementsByTagName(
$tag_name)->item(0);
$element_text = (is_object($tag_node)) ?
$tag_node->nodeValue: "";
if ($element_text) {
break;
}
}
if ($db_element == "link" && $tag_node &&
empty($element_text)) {
$element_text = $tag_node->getAttribute("href");
if (empty($element_text)) {
$element_text = $tag_node->getAttribute("url");
}
if (empty($element_text)) {
$element_text = $tag_node->getAttribute("src");
}
$element_text = UrlParser::canonicalLink($element_text,
$podcast["SOURCE_URL"]);
}
$item[$db_element] = strip_tags($element_text);
}
if (!empty($item['link']) && !empty($item['title']) &&
!empty($item_elements['link'])
&& $item['link'] == $item_elements['link']) {
$item['link'] .= "#" . L\crawlHash($item['title']);
}
if (empty($item['guid'])) {
$item['guid'] = L\crawlHash($item['link'] . $item['title']);
}
if ($test_mode) {
$log_function(print_r($item, true));
$did_add = true;
} else {
$did_add = $this->downloadPodcastItemIfNew($item, $podcast,
$age);
}
if ($did_add) {
$num_added++;
}
$num_seen++;
}
return [$num_added, $test_results];
}
/**
* Given a podcast item from a podcast feed page determines if it has
* been downloaded or not and if not whether it is recent enough to
* download. If it is recent enough, it scrapes the file to download
* and downloads any other intermediate files need to find the
* file to download, then finally downloads this podcast item. If the
* podcast item is built out of multiple videos, it concatenates them
* and makes a single video. It then moves the podcast item to the
* appropriate wiki folder.
*
* @param array $item an associative array about one item on a podcast
* feed page
* @param array &$podcast a reference to an associate array of the podcast
* feed the item is from. This is used for the language etc of the item
* and is also used to store updates to what podcasts have already been
* downloaded
* @param int $age how many seconds ago is still considered a recent
* enough podcast to process
* @return bool whether downloaded or not.
*/
public function downloadPodcastItemIfNew($item, &$podcast, $age)
{
$wiki_model = $this->wiki_model;
if (!empty($podcast["FOLDER_KEY"])) {
self::heartbeatPodcastDownload($podcast["FOLDER_KEY"]);
}
$group_model = $this->group_model;
$controller = new CrawlController(); /* only need for clean() method */
$pubdate = (empty($item['pubdate'])) ? time():
(is_int($item['pubdate']) ? $item['pubdate'] :
strtotime($item['pubdate']));
if ($pubdate + $age < time()) {
L\crawlLog("Podcast Item: {$item['title']} is too old skipping...");
return false;
}
if (key_exists($item['guid'], $podcast["PREVIOUSLY_DOWNLOADED"])) {
/* What is kept under an item's name here is the page the
item points at, not the media file itself. A page is
fetched again to find what media it now carries, and its
contents change, so a run somebody asked for by pressing
the force control fetches it again rather than trusting
the record of what it held before. A run on the timer
keeps skipping what it has already seen. */
if (!$this->request_driven) {
L\crawlLog("Podcast Item: {$item['title']} already ".
"downloaded, skipping...");
return false;
}
L\crawlLog("Podcast Item: {$item['title']} fetching again, " .
"this update was asked for...");
}
$type = UrlParser::getDocumentType($item['link']);
$file_name = UrlParser::getDocumentFilename($item['link']);
if (empty($file_name)) {
$file_name = date("Y-m-d-H-i");
}
if (!empty($type)) {
$file_name .= ".$type";
}
list($group_id, $page_id, $sub_path, $file_pattern) =
$wiki_model->getGroupIdPageIdSubPathFromName($podcast['WIKI_PAGE'],
$podcast['LANGUAGE']);
$file_name = $this->makeFileNamePattern($file_name, $file_pattern,
substr($item['title'], 0, C\NAME_LEN), $pubdate);
$file_name = $controller->clean($file_name, "file_name");
$type = UrlParser::getDocumentType($file_name);
if (!empty($item['link'])) {
$data = $this->downloadPodcastItem($item['link'], $type);
if ($data) {
$wiki_model->copyFileToGroupPageResource("", $file_name,
L\mimeType($file_name), $group_id, $page_id, $sub_path,
$data);
$podcast["PREVIOUSLY_DOWNLOADED"][$item['guid']] = [
"FILENAME" => $file_name,
"SAVE_TIMESTAMP" => time()
];
return true;
}
}
return false;
}
/**
* downloadPodcastItem fetches one item of a podcast and hands back
* its bytes. downloadPodcastItemIfNew calls it once it has decided
* an item is worth keeping.
*
* An ordinary file is fetched as a page and handed back as it
* arrived, with no ffmpeg run on it at all. A playlist naming the
* pieces of a video is handed to ffmpeg, which fetches the pieces
* and joins them into one file with their contents copied as they
* stand. Nothing is written again, so a fetch takes about as long
* as fetching the pieces does.
*
* @param string $url address of the item to fetch
* @param string $type ending the item should be written under, such
* as mp4 or mp3
* @return mixed the item's bytes as a string, or false where nothing
* could be fetched
*/
public function downloadPodcastItem($url, $type = "mp4")
{
L\crawlLog("Downloading ...$url");
$data = $this->getPage($url);
if (!str_starts_with($data, "#EXTM3")) {
return $data;
}
if (!C\nsdefined('FFMPEG') || !C\FFMPEG) {
L\crawlLog("----$url names the pieces of a video, which " .
"needs ffmpeg to join. Set FFMPEG to fetch it.");
return false;
}
return $this->fetchWithFfmpeg($url, $type);
}
/**
* fetchWithFfmpeg has ffmpeg join the pieces a playlist names into
* one file and hands back that file's bytes. downloadPodcastItem
* calls it where an item turns out to be a playlist rather than a
* file.
*
* The pieces are copied as they stand, so this takes about as long
* as fetching them does. The working folder is deleted afterward
* however the fetch went, so a failed one leaves nothing behind.
*
* @param string $url address of the playlist
* @param string $type ending the item is written under, such as mp4
* @return mixed the item's bytes as a string, or false where ffmpeg
* could not join the pieces
*/
private function fetchWithFfmpeg($url, $type)
{
$convert_folder = C\SCHEDULES_DIR . self::CONVERT_FOLDER;
if (!$this->makeFolder($convert_folder)) {
return false;
}
$convert_folder .= "/" . L\crawlHash($url);
if (!$this->makeFolder($convert_folder)) {
return false;
}
$joined = "$convert_folder/joined.$type";
$data = false;
if ($this->runFfmpeg(escapeshellarg($url),
"-c copy -movflags +faststart -loglevel error -stats",
$joined)) {
$data = file_get_contents($joined);
} else {
L\crawlLog("----ffmpeg could not join the pieces of $url");
}
$model = new GroupModel();
$model->db->unlinkRecursive($convert_folder);
return $data;
}
/**
* runFfmpeg runs one ffmpeg command and says whether it left a file
* worth keeping. fetchWithFfmpeg calls it to join the pieces of a
* playlist.
*
* @param string $from what to read, already quoted for a shell
* @param string $settings how to write it, such as -c copy
* @param string $to path to write to
* @return bool whether a file with something in it was written
*/
private function runFfmpeg($from, $settings, $to)
{
$command = C\FFMPEG . " -y -i $from $settings " .
escapeshellarg($to) . " 2>&1";
L\crawlLog($command);
list($said, $went_wrong) = $this->runAndSayWhatIsHappening(
$command);
if ($went_wrong == 0 && file_exists($to) && filesize($to) > 0) {
return true;
}
L\crawlLog(implode("\n", array_slice($said,
-self::FFMPEG_LINES_LOGGED)));
return false;
}
/**
* runAndSayWhatIsHappening runs a command and passes what it says
* along as it says it, rather than waiting for it to finish.
* fetchWithFfmpeg calls it, since writing a long video can take
* minutes and a media updater that says nothing for that long looks
* to a watcher like one that has died.
*
* Where the updater is writing to a log, ffmpeg's own progress goes
* there, which is what a watcher of ManageMachines reads. Where it
* was started from a terminal, the same lines go to the terminal.
* Both follow from crawlLog, so neither is decided here.
*
* Lines are passed along no more often than SECONDS_BETWEEN_SAYINGS
* apart, since ffmpeg writes a progress line several times a second
* and every one of them in a log would bury everything else.
*
* @param string $command what to run, with its error output already
* folded into what it says
* @return array the lines the command said, and the number it
* finished with, in that order
*/
private function runAndSayWhatIsHappening($command)
{
$running = popen($command, "r");
if (!is_resource($running)) {
return [[], 1];
}
$said = [];
$last_said_at = 0;
$part = "";
/* ffmpeg ends each line of its progress with a carriage return
and no newline, so that a terminal draws the next one over the
last. Reading whole lines would therefore wait for the run to
finish, so what arrives is read in pieces and broken on either
mark. */
while (!feof($running)) {
$piece = fread($running, self::BYTES_READ_AT_ONCE);
if ($piece === false || $piece === "") {
break;
}
$part .= $piece;
$lines = preg_split('/[\r\n]+/', $part);
$part = array_pop($lines);
foreach ($lines as $line) {
if (trim($line) === "") {
continue;
}
$said[] = $line;
$now = microtime(true);
if ($now - $last_said_at >=
self::SECONDS_BETWEEN_SAYINGS) {
L\crawlLog("----$line");
$last_said_at = $now;
}
}
}
if (trim($part) !== "") {
$said[] = $part;
}
$went_wrong = pclose($running);
return [$said, $went_wrong];
}
/**
* Used to construct a filename for a downloaded podcast item suitable
* to be used when stored in a wiki page's resource folder
*
* @param string $file_name name of file
* @param string $file_pattern string which can contain %F for previous
* filename, %T for title, and date %date_command, for example,
* %Y for year, %m for month, %d for day, etc. These will be substituted
* with their values when wriitng out the wiki name for the downloaded
* podcast item.
* @param string $title a title string for wiki item
* @param int $pubdate when the wiki item was published as a Unix timestamp.
* The value of this is used when computing values for the $file_pattern
* @return string output filename for wiki item
*/
private function makeFileNamePattern($file_name, $file_pattern,
$title = "", $pubdate = null)
{
$translates = ["'" => "", "\"" => "", "/" => "-", "|" => "-",
'$' => '',];
$file_name_parts = explode("?", $file_name);
$file_name = strtr(basename($file_name_parts[0]), $translates);
$title = strtr($title, $translates);
if (empty($file_pattern)) {
return $file_name;
}
if (!$pubdate) {
$pubdate = time();
}
$pattern_parts = preg_split('/(\%\w)/u', $file_pattern, -1,
PREG_SPLIT_DELIM_CAPTURE);
$out_name = "";
foreach ($pattern_parts as $pattern_part) {
if (!empty($pattern_part[0]) && $pattern_part[0] == '%') {
if ($pattern_part[1] == 'F') {
$out_name .= $file_name;
} else if ($pattern_part[1] == 'T') {
$out_name .= $title;
} else {
$out_name .= date($pattern_part[1], $pubdate);
}
} else {
$out_name .= $pattern_part;
}
}
/* just in case the name came from a url with extra garbbage */
return $out_name;
}
/**
* Makes a directory in a way compatible with yioop's error handling.
*
* @param string $folder name of directory/folder to create.
* @return boolean whether directory was created
*/
private function makeFolder($folder)
{
if (!file_exists($folder)) {
set_error_handler(null);
@mkdir($folder);
restore_error_handler();
if (!file_exists($folder)) {
L\crawlLog("----Unable to create folder. Bailing!");
return false;
}
}
return true;
}
/**
* Downloads the internet page with the give url.
*
* @param $url The url want to download
* @return string contents of downloaded page
*/
private function getPage($url)
{
return FetchUrl::getPage($url, null, false, null,
4 * C\SINGLE_PAGE_TIMEOUT);
}
}
(c) 2026 Seekquarry - Open Source Search Engine Software