diff --git a/interface/lib/classes/aps_crawler.inc.php b/interface/lib/classes/aps_crawler.inc.php index 21a21710b79d7e2c1477438b4ec617d0e127a3cf..4a6409227ea018a2e22ac8b745516f8511203327 100644 --- a/interface/lib/classes/aps_crawler.inc.php +++ b/interface/lib/classes/aps_crawler.inc.php @@ -189,7 +189,8 @@ class ApsCrawler extends ApsBase curl_setopt($conn[$i], CURLOPT_TIMEOUT, 0); curl_setopt($conn[$i], CURLOPT_FAILONERROR, 1); curl_setopt($conn[$i], CURLOPT_FOLLOWLOCATION, 1); - curl_setopt($conn[$i], CURLOPT_SSL_VERIFYPEER, 0); + curl_setopt($conn[$i], CURLOPT_SSL_VERIFYHOST, 1); + curl_setopt($conn[$i], CURLOPT_SSL_VERIFYPEER, false); curl_multi_add_handle($mh, $conn[$i]); } @@ -284,179 +285,185 @@ class ApsCrawler extends ApsBase $apps_count = substr_count($apps[$j], '0'); if($apps_count == 0) // obviously this vendor provides one or more apps { - // Rename namespaces and register them - $xml = str_replace("xmlns=", "ns=", $apps[$j]); - $sxe = new SimpleXMLElement($xml); - $namespaces = $sxe->getDocNamespaces(true); - foreach($namespaces as $ns => $url) $sxe->registerXPathNamespace($ns, $url); + try { + // Rename namespaces and register them + $xml = str_replace("xmlns=", "ns=", $apps[$j]); + $sxe = new SimpleXMLElement($xml); + $namespaces = $sxe->getDocNamespaces(true); + foreach($namespaces as $ns => $url) $sxe->registerXPathNamespace($ns, $url); - //Find highest version - $app_version = "0.0.0"; - $entry_pos = 1; - for ($p = 1; ; $p++) { - $app_version_tmp = parent::getXPathValue($sxe, 'entry[position()=' . $p . ']/a:version'); - if (strlen($app_version_tmp) < 1) break; - if (version_compare($app_version_tmp, $app_version) >= 0) { - $app_version = $app_version_tmp; - $entry_pos = $p; + //Find highest version + $app_version = "0.0.0"; + $entry_pos = 1; + for ($p = 1; ; $p++) { + $app_version_tmp = parent::getXPathValue($sxe, 'entry[position()=' . $p . ']/a:version'); + if (strlen($app_version_tmp) < 1) break; + if (version_compare($app_version_tmp, $app_version) >= 0) { + $app_version = $app_version_tmp; + $entry_pos = $p; + } } - } - // Fetching values of interest - //$app_name = parent::getXPathValue($sxe, 'entry[position()=1]/a:name'); - //$app_version = parent::getXPathValue($sxe, 'entry[position()=1]/a:version'); - //$app_release = parent::getXPathValue($sxe, 'entry[position()=1]/a:release'); - $app_name = parent::getXPathValue($sxe, "entry[position()=" . $entry_pos . "]/a:name"); - $app_version = parent::getXPathValue($sxe, "entry[position()=" . $entry_pos . "]/a:version"); - $app_release = parent::getXPathValue($sxe, "entry[position()=" . $entry_pos . "]/a:release"); - - // Find out a (possibly) existing package version - $ex_ver = ''; - /* - array_walk($existing_apps, - create_function('$v, $k, $ex_ver', 'if($v["Name"] == "'.$app_name.'") $ex_ver = $v["CurrentVersion"];'), &$ex_ver); - */ - if(is_array($existing_apps)) { - foreach($existing_apps as $k => $v) { - if($v["Name"] == $app_name) $ex_ver = $v["CurrentVersion"]; + // Fetching values of interest + //$app_name = parent::getXPathValue($sxe, 'entry[position()=1]/a:name'); + //$app_version = parent::getXPathValue($sxe, 'entry[position()=1]/a:version'); + //$app_release = parent::getXPathValue($sxe, 'entry[position()=1]/a:release'); + $app_name = parent::getXPathValue($sxe, "entry[position()=" . $entry_pos . "]/a:name"); + $app_version = parent::getXPathValue($sxe, "entry[position()=" . $entry_pos . "]/a:version"); + $app_release = parent::getXPathValue($sxe, "entry[position()=" . $entry_pos . "]/a:release"); + + // Find out a (possibly) existing package version + $ex_ver = ''; + /* + array_walk($existing_apps, + create_function('$v, $k, $ex_ver', 'if($v["Name"] == "'.$app_name.'") $ex_ver = $v["CurrentVersion"];'), &$ex_ver); + */ + if(is_array($existing_apps)) { + foreach($existing_apps as $k => $v) { + if($v["Name"] == $app_name) $ex_ver = $v["CurrentVersion"]; + } } - } - $new_ver = $app_version.'-'.$app_release; - $local_intf_folder = $this->interface_pkg_dir.'/'.$app_name.'-'.$new_ver.'.app.zip/'; - - // Proceed if a newer or at least equal version has been found with server mode or - // interface mode is activated and there are no valid APP-META.xml and PKG_URL existing yet - if((!$this->interface_mode && version_compare($new_ver, $ex_ver) >= 0) || ($this->interface_mode && (!file_exists($local_intf_folder.'APP-META.xml') || filesize($local_intf_folder.'APP-META.xml') == 0 || !file_exists($local_intf_folder.'PKG_URL') || filesize($local_intf_folder.'PKG_URL') == 0))){ - // Check if we already have an old version of this app - if(!empty($ex_ver) && version_compare($new_ver, $ex_ver) == 1) $apps_updated++; - - //$app_dl = parent::getXPathValue($sxe, "entry[position()=1]/link[@a:type='aps']/@href"); - //$app_filesize = parent::getXPathValue($sxe, "entry[position()=1]/link[@a:type='aps']/@length"); - //$app_metafile = parent::getXPathValue($sxe, "entry[position()=1]/link[@a:type='meta']/@href"); - $app_dl = parent::getXPathValue($sxe, "entry[position()=" . $entry_pos . "]/link[@a:type='aps']/@href"); - $app_filesize = parent::getXPathValue($sxe, "entry[position()=" . $entry_pos . "]/link[@a:type='aps']/@length"); - $app_metafile = parent::getXPathValue($sxe, "entry[position()=" . $entry_pos . "]/link[@a:type='meta']/@href"); - - //$this->app_download_url_list[$app_name.'-'.$new_ver.'.app.zip'] = $app_dl; - // Skip ASP.net packages because they can't be used at all - $asp_handler = parent::getXPathValue($sxe, '//aspnet:handler'); - $asp_permissions = parent::getXPathValue($sxe, '//aspnet:permissions'); - $asp_version = parent::getXPathValue($sxe, '//aspnet:version'); - if(!empty($asp_handler) || !empty($asp_permissions) || !empty($asp_version)) continue; - - // Interface mode (download only parts) - if($this->interface_mode) - { - // Delete an obviously out-dated version from the system and DB - if(!empty($ex_ver) && version_compare($new_ver, $ex_ver) == 1) + $new_ver = $app_version.'-'.$app_release; + $local_intf_folder = $this->interface_pkg_dir.'/'.$app_name.'-'.$new_ver.'.app.zip/'; + + // Proceed if a newer or at least equal version has been found with server mode or + // interface mode is activated and there are no valid APP-META.xml and PKG_URL existing yet + if((!$this->interface_mode && version_compare($new_ver, $ex_ver) >= 0) || ($this->interface_mode && (!file_exists($local_intf_folder.'APP-META.xml') || filesize($local_intf_folder.'APP-META.xml') == 0 || !file_exists($local_intf_folder.'PKG_URL') || filesize($local_intf_folder.'PKG_URL') == 0))){ + // Check if we already have an old version of this app + if(!empty($ex_ver) && version_compare($new_ver, $ex_ver) == 1) $apps_updated++; + + //$app_dl = parent::getXPathValue($sxe, "entry[position()=1]/link[@a:type='aps']/@href"); + //$app_filesize = parent::getXPathValue($sxe, "entry[position()=1]/link[@a:type='aps']/@length"); + //$app_metafile = parent::getXPathValue($sxe, "entry[position()=1]/link[@a:type='meta']/@href"); + $app_dl = parent::getXPathValue($sxe, "entry[position()=" . $entry_pos . "]/link[@a:type='aps']/@href"); + $app_filesize = parent::getXPathValue($sxe, "entry[position()=" . $entry_pos . "]/link[@a:type='aps']/@length"); + $app_metafile = parent::getXPathValue($sxe, "entry[position()=" . $entry_pos . "]/link[@a:type='meta']/@href"); + + //$this->app_download_url_list[$app_name.'-'.$new_ver.'.app.zip'] = $app_dl; + // Skip ASP.net packages because they can't be used at all + $asp_handler = parent::getXPathValue($sxe, '//aspnet:handler'); + $asp_permissions = parent::getXPathValue($sxe, '//aspnet:permissions'); + $asp_version = parent::getXPathValue($sxe, '//aspnet:version'); + if(!empty($asp_handler) || !empty($asp_permissions) || !empty($asp_version)) continue; + + // Interface mode (download only parts) + if($this->interface_mode) { - $old_folder = $this->interface_pkg_dir.'/'.$app_name.'-'.$ex_ver.'.app.zip'; - if(file_exists($old_folder)) $this->removeDirectory($old_folder); - - /* - $app->db->query("UPDATE aps_packages SET package_status = '".PACKAGE_OUTDATED."' WHERE name = '". - $app->db->quote($app_name)."' AND CONCAT(version, '-', CAST(`release` AS CHAR)) = '". - $app->db->quote($ex_ver)."';"); - */ - $tmp = $app->db->queryOneRecord("SELECT id FROM aps_packages WHERE name = '". - $app->db->quote($app_name)."' AND CONCAT(version, '-', CAST(`release` AS CHAR)) = '". - $app->db->quote($ex_ver)."';"); - $app->db->datalogUpdate('aps_packages', "package_status = ".PACKAGE_OUTDATED, 'id', $tmp['id']); - unset($tmp); - } - - // Create the local folder if not yet existing - if(!file_exists($local_intf_folder)) @mkdir($local_intf_folder, 0777, true); + // Delete an obviously out-dated version from the system and DB + if(!empty($ex_ver) && version_compare($new_ver, $ex_ver) == 1) + { + $old_folder = $this->interface_pkg_dir.'/'.$app_name.'-'.$ex_ver.'.app.zip'; + if(file_exists($old_folder)) $this->removeDirectory($old_folder); + + /* + $app->db->query("UPDATE aps_packages SET package_status = '".PACKAGE_OUTDATED."' WHERE name = '". + $app->db->quote($app_name)."' AND CONCAT(version, '-', CAST(`release` AS CHAR)) = '". + $app->db->quote($ex_ver)."';"); + */ + $tmp = $app->db->queryOneRecord("SELECT id FROM aps_packages WHERE name = '". + $app->db->quote($app_name)."' AND CONCAT(version, '-', CAST(`release` AS CHAR)) = '". + $app->db->quote($ex_ver)."';"); + $app->db->datalogUpdate('aps_packages', "package_status = ".PACKAGE_OUTDATED, 'id', $tmp['id']); + unset($tmp); + } - // Save the package URL in an extra file because it's not part of the APP-META.xml file - @file_put_contents($local_intf_folder.'PKG_URL', $app_dl); + // Create the local folder if not yet existing + if(!file_exists($local_intf_folder)) @mkdir($local_intf_folder, 0777, true); - // Download the meta file - $local_metafile = $local_intf_folder.'APP-META.xml'; - if(!file_exists($local_metafile) || filesize($local_metafile) == 0) - { - $apps_to_dl[] = array('name' => 'APP-META.xml', - 'url' => $app_metafile, - 'filesize' => 0, - 'localtarget' => $local_metafile); - $apps_downloaded++; - } + // Save the package URL in an extra file because it's not part of the APP-META.xml file + @file_put_contents($local_intf_folder.'PKG_URL', $app_dl); - // Download package license - //$license = parent::getXPathValue($sxe, "entry[position()=1]/link[@a:type='eula']/@href"); - $license = parent::getXPathValue($sxe, "entry[position()=" . $entry_pos . "]/link[@a:type='eula']/@href"); - if($license != '') - { - $local_license = $local_intf_folder.'LICENSE'; - if(!file_exists($local_license) || filesize($local_license) == 0) + // Download the meta file + $local_metafile = $local_intf_folder.'APP-META.xml'; + if(!file_exists($local_metafile) || filesize($local_metafile) == 0) { - $apps_to_dl[] = array('name' => basename($license), - 'url' => $license, + $apps_to_dl[] = array('name' => 'APP-META.xml', + 'url' => $app_metafile, 'filesize' => 0, - 'localtarget' => $local_license); + 'localtarget' => $local_metafile); + $apps_downloaded++; } - } - // Download package icon - //$icon = parent::getXPathValue($sxe, "entry[position()=1]/link[@a:type='icon']/@href"); - $icon = parent::getXPathValue($sxe, "entry[position()=" . $entry_pos . "]/link[@a:type='icon']/@href"); - if($icon != '') - { - $local_icon = $local_intf_folder.basename($icon); - if(!file_exists($local_icon) || filesize($local_icon) == 0) + // Download package license + //$license = parent::getXPathValue($sxe, "entry[position()=1]/link[@a:type='eula']/@href"); + $license = parent::getXPathValue($sxe, "entry[position()=" . $entry_pos . "]/link[@a:type='eula']/@href"); + if($license != '') { - $apps_to_dl[] = array('name' => basename($icon), - 'url' => $icon, - 'filesize' => 0, - 'localtarget' => $local_icon); + $local_license = $local_intf_folder.'LICENSE'; + if(!file_exists($local_license) || filesize($local_license) == 0) + { + $apps_to_dl[] = array('name' => basename($license), + 'url' => $license, + 'filesize' => 0, + 'localtarget' => $local_license); + } } - } - // Download available screenshots - //$screenshots = parent::getXPathValue($sxe, "entry[position()=1]/link[@a:type='screenshot']", true); - $screenshots = parent::getXPathValue($sxe, "entry[position()=" . $entry_pos . "]/link[@a:type='screenshot']", true); - if(!empty($screenshots)) - { - foreach($screenshots as $screen) + // Download package icon + //$icon = parent::getXPathValue($sxe, "entry[position()=1]/link[@a:type='icon']/@href"); + $icon = parent::getXPathValue($sxe, "entry[position()=" . $entry_pos . "]/link[@a:type='icon']/@href"); + if($icon != '') { - $local_screen = $local_intf_folder.basename($screen['href']); - if(!file_exists($local_screen) || filesize($local_screen) == 0) + $local_icon = $local_intf_folder.basename($icon); + if(!file_exists($local_icon) || filesize($local_icon) == 0) { - $apps_to_dl[] = array('name' => basename($screen['href']), - 'url' => $screen['href'], + $apps_to_dl[] = array('name' => basename($icon), + 'url' => $icon, 'filesize' => 0, - 'localtarget' => $local_screen); + 'localtarget' => $local_icon); + } + } + + // Download available screenshots + //$screenshots = parent::getXPathValue($sxe, "entry[position()=1]/link[@a:type='screenshot']", true); + $screenshots = parent::getXPathValue($sxe, "entry[position()=" . $entry_pos . "]/link[@a:type='screenshot']", true); + if(!empty($screenshots)) + { + foreach($screenshots as $screen) + { + $local_screen = $local_intf_folder.basename($screen['href']); + if(!file_exists($local_screen) || filesize($local_screen) == 0) + { + $apps_to_dl[] = array('name' => basename($screen['href']), + 'url' => $screen['href'], + 'filesize' => 0, + 'localtarget' => $local_screen); + } } } } - } - else // Server mode (download whole ZIP archive) - { - // Delete an obviously out-dated version from the system - if(!empty($ex_ver) && version_compare($new_ver, $ex_ver) == 1) - { - $old_file = $this->packages_dir.'/'.$app_name.'-'.$ex_ver.'.app.zip'; - if(file_exists($old_file)) $this->removeDirectory($old_file); - } + else // Server mode (download whole ZIP archive) + { + // Delete an obviously out-dated version from the system + if(!empty($ex_ver) && version_compare($new_ver, $ex_ver) == 1) + { + $old_file = $this->packages_dir.'/'.$app_name.'-'.$ex_ver.'.app.zip'; + if(file_exists($old_file)) $this->removeDirectory($old_file); + } - // Attention: $new_ver can also be == $ex_ver (according to version_compare >= 0) - $local_zip = $this->packages_dir.'/'.$app_name.'-'.$new_ver.'.app.zip'; + // Attention: $new_ver can also be == $ex_ver (according to version_compare >= 0) + $local_zip = $this->packages_dir.'/'.$app_name.'-'.$new_ver.'.app.zip'; - // Before re-downloading a file, make sure it's not yet existing on HDD (due to DB inconsistency) - if((file_exists($local_zip) && (filesize($local_zip) == $app_filesize)) === false) - { - $apps_to_dl[] = array('name' => $app_name, - 'url' => $app_dl, - 'filesize' => $app_filesize, - 'localtarget' => $local_zip); - $apps_downloaded++; + // Before re-downloading a file, make sure it's not yet existing on HDD (due to DB inconsistency) + if((file_exists($local_zip) && (filesize($local_zip) == $app_filesize)) === false) + { + $apps_to_dl[] = array('name' => $app_name, + 'url' => $app_dl, + 'filesize' => $app_filesize, + 'localtarget' => $local_zip); + $apps_downloaded++; + } } } - } - unset($sxe); - $apps_in_repo++; + unset($sxe); + $apps_in_repo++; + } catch (Exception $e) { + // We dont want the crawler to fail on xml parse errors + $app->log($this->log_prefix.$e->getMessage(), LOGLEVEL_WARN); + //echo 'Caught exception: ', $e->getMessage(), "\n"; + } } } //var_dump($apps);