ImportExternalImages.module

<?php namespace ProcessWire;

/**
 *
 * Import External Images
 * Enables pages to import images in pasted content, into the local images field.
 *
 * @author Macrura / outflux3
 *
 * ProcessWire 2.x
 * Copyright (C) 2014 by Ryan Cramer
 * Licensed under GNU/GPL v2, see LICENSE.TXT
 *
 * http://processwire.com
 *
 *  @todo
 * 1) setting for external base URL (temp)
 * 2) enable/disable (without uninstall)
 * 3) role based permissions
 *
 */

class ImportExternalImages extends WireData implements Module, ConfigurableModule {

	public static function getModuleInfo() {

		return array(
			'title' 	=> 'Import External Images',
			'version' 	=> '2.0.15',
			'summary' 	=> 'For content pasted in textarea fields, this will import external images to the images field.',
			'href' 		=> 'https://processwire.com/talk/topic/15538-import-external-images/',
			'singular' 	=> true,
			'autoload' 	=> 'template=admin',
			'icon' 		=> 'download',
			'author'   => 'Marc W. | @Macrura(community), @outflux3 (Github), http://nibiri.com',
			);
	}


	protected static $configDefaults = array(
		'img_field'			=> 'images',
		'ta_field'			=> 'body',
		'url_field'			=> '',
		'proceed'			=> false,
		'dont_wrap_figure' 	=> '',
	);


	/**
	 * Data as used by the get/set functions
	 *
	 */
	protected $data = array();

	/**
	 * Set our configuration defaults
	 *
	 */
	public function __construct() {
		foreach(self::$configDefaults as $key => $value) {
			$this->set($key, $value);
		}
	}


	/**
	 * Module configuration screen
	 *
	 */
	public static function getModuleConfigInputfields(array $data) {

		foreach(self::$configDefaults as $key => $value) {
			if(!isset($data[$key])||$data[$key]=="") $data[$key] = $value;
		}

		unset($data['cacheClear']);
		$inputfields = new InputfieldWrapper();

		// INSTRUCTIONS
        // ------------------------------------------------------------------------
	    $f = wire('modules')->get('InputfieldMarkup');
        $f->name  = 'instructions';
        $f->label = __('Instructions', __FILE__);
        $f->markupText = __('For this module to work, please select the fields to allow this module to perform replacements on, as well as the name of the image field on the same page to add the images to.', __FILE__);
        $f->markupText .= "<br>";
        $f->markupText .= __('<span style="font-style:italic;color:red">If you are importing images from a 3rd party site, make sure you have permissions to store and display those images on your site.</span>', __FILE__);
        $inputfields->add($f);


		// ENABLE FOR TEMPLATES
        // ------------------------------------------------------------------------
		$f = wire('modules')->get('InputfieldAsmSelect');
		$f->attr('name+id', 'enabledTemplates');
		$f->label = __('Enabled templates', __FILE__);
		$f->description = __('URL Scanning will occur only on the selected templates. If none are selected, scanning will occur on all templates.', __FILE__);
		$f->attr('title', __('Enabled templates', __FILE__));
		$f->setAsmSelectOption('sortable', false);

		// populate with all available templates
		foreach(wire('templates') as $t) {
			// filter out system templates
			if(!($t->flags & Template::flagSystem)) $f->addOption($t->name);

			// breaking change - need to have an upgrade method before this change (?)
			//if(!($t->flags & Template::flagSystem)) $f->addOption($t->id, $t->name);
		}
		if(isset($data['enabledTemplates'])) $f->value = $data['enabledTemplates'];
		$f->columnWidth = 25;
		$inputfields->add($f);


        // ------------------------------------------------------------------------
        // Field to put images into
        // ------------------------------------------------------------------------
        $img_fields = wire('fields')->find('type=FieldtypeImage|FieldtypeCroppableImage3');
        if (count($img_fields) > 0) {
            $f              = wire('modules')->get('InputfieldSelect');
            $f->name        = 'img_field';
            $f->label       = __('Images Field', __FILE__);

            foreach ($img_fields as $img_field) {
                $f->addOption($img_field->name, self::getFormattedOptionName($img_field));
            }

            if ($data[ $f->name ]) {
                $f->attr('value', $data[ $f->name ]);
            }
        } else {
            // add info that no URL type fields are available
            $f        = wire('modules')->get('InputfieldMarkup');
            $f->name  = 'no_images_fields';
            $f->label = __('Enabled fields', __FILE__);
            $f->markupText = __('There are no image fields available', __FILE__);
        }
        $f->columnWidth = 25;
        $inputfields->add($f);

        // ------------------------------------------------------------------------
        // Textarea Field to search for external image URLs
        // ------------------------------------------------------------------------
        $ta_fields = wire('fields')->find('type=FieldtypeTextarea|FieldtypeTextareaLanguage');
        if (count($ta_fields) > 0) {
            $f              = wire('modules')->get('InputfieldSelect');
            $f->name        = 'ta_field';
            $f->label       = __('Textarea Field', __FILE__);
            $f->description = __('Select the field to scan for external URLs.', __FILE__);
            foreach ($ta_fields as $ta_field) {
                $f->addOption($ta_field->name, self::getFormattedOptionName($ta_field));
            }

            if ($data[ $f->name ]) {
                $f->attr('value', $data[ $f->name ]);
            }
        } else {
            // add info that no URL type fields are available
            $f        = wire('modules')->get('InputfieldMarkup');
            $f->name  = 'no_textarea_fields';
            $f->label = __('Textarea fields', __FILE__);
            $f->markupText = __('There are no textarea fields available to search in.', __FILE__);
        }
        $f->columnWidth = 25;
        $inputfields->add($f);


        // ------------------------------------------------------------------------
        // URL Field to search for external image -> new feature for getting single images.
        // ------------------------------------------------------------------------
        $url_fields = wire('fields')->find('type=FieldtypeURL');
        if (count($url_fields) > 0) {
            $f              = wire('modules')->get('InputfieldSelect');
            $f->name        = 'url_field';
            $f->label       = __('URL Field', __FILE__);
            $f->description = __('Select a URL field to use for fetching an external image.', __FILE__);
            foreach ($url_fields as $url_field) {
                $f->addOption($url_field->name, self::getFormattedOptionName($url_field));
            }

            if ($data[ $f->name ]) {
                $f->attr('value', $data[ $f->name ]);
            }
        } else {
            // add info that no URL type fields are available
            $f        = wire('modules')->get('InputfieldMarkup');
            $f->name  = 'no_url_fields';
            $f->label = __('URL fields', __FILE__);
            $f->markupText = __('There are no URL fields available for image fetching.', __FILE__);
        }
        $f->columnWidth = 25;
        $inputfields->add($f);

		// ------------------------------------------------------------------------
        // Should images be wrapped in a figure tag?
        // ------------------------------------------------------------------------

		$f = wire('modules')->get('InputfieldCheckbox');
		$f->attr('name', 'dont_wrap_figure');
		$f->attr('value', 1);
		$f->label = __('Prevent wrapping images in figure tag', __FILE__);
		if($data[$f->name]) $f->attr('checked', 'checked');

		$f->columnWidth = 100;
		$inputfields->add($f);

		// ------------------------------------------------------------------------
		return $inputfields;
	}



    public static function getFormattedOptionName($obj) {
        return !empty($obj->label) ? $obj->label . ' (' . $obj->name . ')' : $obj->name;
    }

	/**
	 * Initialize the module
	 *
	 * ProcessWire calls this when the module is loaded. For 'autoload' modules, this will be called
	 * when ProcessWire's API is ready. As a result, this is a good place to attach hooks.
	 *
	 */
	public function init() {
		$this->pages->addHookAfter('save', $this, 'importExternalImages');

	}

	/**
	 * ready() is called when both PW's API is ready and $page object is initialized (for autoload modules!)
	 * Add the hook here to be able to skip it based on the template of the edited page.
	 *
	 */
	public function ready() {
		// we're interested in page editor only
		if(wire('page')->process != 'ProcessPageEdit') return;

		// skip changing templates (only target the actual edit form)
		$id = (int)$this->input->get('id');
		if(!$id) return;

		// wire('page') would be the page with ProcessPageEdit
		// GET parameter id tells the page that's being edited
		$this->editedPage = wire('pages')->get($id);

		// don't even consider system templates
		if($this->editedPage->template->flags & Template::flagSystem) return;

		// hook only if
		// 1) no templates have been chosen (=all enabled) OR
		// 2) the template of the edited page has been chosen
		if(count($this->enabledTemplates) == 0 or
		   in_array($this->editedPage->template->name, $this->enabledTemplates)) {
			$this->proceed = true;
		}
	}



	/**
	 * [importExternalImages description]
	 * @param  [type] $event [description]
	 * @return [type]        [description]
	 */
	public function importExternalImages($event) {

		if(!$this->proceed) return;

		$page = $event->arguments[0];

		$ta_field 	= $this->ta_field;
		$img_field 	= $this->img_field;

		$dont_wrap_figure = $this->dont_wrap_figure;

		$field_image = $this->wire('fields')->get($img_field);
		$allowedExts = explode(' ', $field_image->extensions);

		// -------------- New feature: Use a field
		$url_field  = $this->url_field;
		if($url_field && $page->$img_field && $page->$url_field && $page->$url_field != '') {

			$img_url = $page->$url_field;
			if(!filter_var($img_url, FILTER_VALIDATE_URL)) {
				$this->error("Invalid URL.");
			}

			// Check to see if the image has the correct file extension
			$imgExtParts = pathinfo($img_url);
			//$imgExt = $imgExtParts['extension'];
			// new - account for ALL CAPS extensions?
			$imgExt = strtolower($imgExtParts['extension']);
			if(!in_array($imgExt, $allowedExts)) {
				$page->of(false);
				$page->$url_field = '';
				$page->save($url_field);
				$this->error("File extension $imgExt not permitted.");
				return;
			}

			// Assuming the extension was ok, proceed
			if(filter_var($img_url, FILTER_VALIDATE_URL)) {
				$page->of(false);
				$page->$img_field->add($img_url);
				$page->$url_field = '';
				$page->save($img_field);
				$page->save($url_field);
				$this->message("Image URL successfully imported.");

				// make sure that any added images have the correct extension:
				$map = array( IMAGETYPE_JPEG=>'jpg', IMAGETYPE_PNG=>'png', IMAGETYPE_GIF=>'gif', IMAGETYPE_WEBP=>'webp' );

				$img = $page->$img_field->first();
				$ext_im = $map[exif_imagetype($img->filename)];
				$addon = false;
				if($img->ext) {
					$ext_fn = strtolower($img->ext);
				} else {
					// image has no extension, add a new extension
					$ext_fn = '';
					$addon = true;
				}
				// rename the image if the extention is mismatched, or there is no extension
				if($ext_im != $ext_fn) {
					// if analysis of local image reveals that it is a webp, with jpg extension, remove the file
					// to prevent imagesizer from crashing.
					if($ext_im == 'webp') {
						$page->$img_field->delete($img);
						$page->save();
						$this->error("Image not imported: extension mismatch - image is a webp.");
					} else {
						if($addon) $newFilename = $sanitizer->pageName($page->title) . '.' . $ext_im;
						if(!$addon) $newFilename = str_replace('.' . $img->ext, '.' . $ext_im, $img->name);
						$img->rename($newFilename);
						$page->save($img_field);
					}
				}

			}
		}
		// -------------- end new feature beta

		if(!$page->$img_field || !$page->$ta_field) return;

		$html = $page->$ta_field;
		$html = preg_replace('/(.*)(.jpe?g|.gif|.png)(\?[\d]+)(.*)/', '$1$2$4', $html);
		if (strpos($html,'<img') === false) return; //return early if no images are embedded in html

        $dom = new \DOMDocument();
        @$dom->loadHTML(mb_convert_encoding($html, 'HTML-ENTITIES', 'UTF-8'));
        $images = $dom->getElementsByTagName('img');
        if(!$images->length) return; // not needed?

        $assetsPath = $this->pages->get($page->id)->filesManager()->url();

        $extCount = 0;
		$errors = [];
		//$messages = [];
		foreach ($images as $image) {
            $img_url = $image->getAttribute('src');

			if(!filter_var($img_url, FILTER_VALIDATE_URL)) continue;

			// added @adrian
			$parsed = parse_url($img_url);
			$img_url= $parsed['scheme']. '://'. $parsed['host']. $parsed['path'];

			// Check to see if the image has an allowed file extension for the field.
			$imgExtParts = pathinfo($img_url);
			$imgExt = strtolower($imgExtParts['extension']);
			if(!in_array($imgExt, $allowedExts)) {
				$errors[] = "File extension $imgExt not permitted for $img_url.";
				continue;
			}

			// If we made it this far, add the image.
			// =================================================================
            $page->$img_field->add($img_url);
			//$messages[] = "Image successfully added: $img_url.";

			// @adrian
			if(strpos($img_url, '%20') !== false) {
				$cleanedName = str_replace('_20', '-', $page->$img_field->last()->filename);
				$page->$img_field->last()->rename($cleanedName);
			}

	        if($image->getAttribute('title') != ''){
	            $page->$img_field->last()->description = $image->getAttribute('title');
	        }
	        elseif($image->getAttribute('alt') != '' && $image->getAttribute('alt') != basename($image->getAttribute('src'))){
	            $page->$img_field->last()->description = $image->getAttribute('alt');
			}

            //resize image to make version to match the size originally in the RTE
            //check to make sure size is different to downloaded version before resizing
			$imgWidth = $image->getAttribute('width');
			if(strpos($imgWidth,'%') !== false) $imgWidth = ''; // don't resize if image is % size.
			if($imgWidth && $imgWidth != $page->$img_field->last()->width) {
                $imgForRte = $page->$img_field->last()->size($imgWidth, 0);
            }
            else {
                $imgForRte = $page->$img_field->last();
            }
			$image->setAttribute('src', $imgForRte->url);

			// added @adrian
			// add check by tobaco
			if(!$dont_wrap_figure) {
				$figure = $dom->createElement('figure');
				$image->parentNode->replaceChild($figure, $image);
				$figure->appendChild($image);
				if($image->getAttribute('alt') != '' && $image->getAttribute('alt') != basename($image->getAttribute('src'))) {
					$figcaption = $dom->createElement('figcaption');
					$figcaption->nodeValue = $image->getAttribute('alt');
					$figure->appendChild($figcaption);
				}
			}

            $extCount++;
        }

        if(!$extCount) return;
    	$page->of(false);

   		$page->$ta_field = preg_replace('/^<!DOCTYPE.+?>/', '', str_replace( array('<html>', '</html>', '<body>', '</body>'), array('', '', '', ''), $dom->saveHTML()));

   	 	$page->save($ta_field);
   	 	$this->message("Image links updated to local images.");

        $page->save($img_field);
        $this->message("External images imported to page.");

		if(count($errors)) {
			$msgOut = "Errors occured and some images were not imported. See below.\n";
			$msgOut .= implode("\n", $errors);
			$this->error($msgOut);
		}

    }


	/**
	 * Method for API Usage of the importer function
	 * @param  [type] $page [Must be instance of ProcessWire\Page ]
	 * @return [type]        [description]
	 */
	/**
	 * API method: import external images from a textarea/RTE field into an images field.
	 *
	 * @param Page        $page
	 * @param string|null $taField  Override the configured textarea field to scan (optional)
	 * @param string|null $imgField Override the configured images field to receive images (optional)
	 */
	public function importImages($page, $taField = null, $imgField = null) {

		$ta_field 	= $taField  ?? $this->ta_field;
		$img_field 	= $imgField ?? $this->img_field;
		$url_field  = $this->url_field;

		$dont_wrap_figure = $this->dont_wrap_figure;

		if($url_field && $page->$img_field && $page->$url_field && $page->$url_field != '') {
			$img_url = $page->$url_field;
			if(filter_var($img_url, FILTER_VALIDATE_URL)) {
				$page->of(false);
				$page->$img_field->add($img_url);
				$page->$url_field = '';
				$page->save($img_field);
				$page->save($url_field);
				$this->message("Image URL successfully imported.");
			}
		}

		if(!$page->fields->has($img_field) || !$page->fields->has($ta_field)) return;

		$html = $page->$ta_field;
		$html = preg_replace('/(.*)(.jpe?g|.gif|.png)(\?[\d]+)(.*)/', '$1$2$4', $html);
		if (strpos($html,'<img') === false) return; //return early if no images are embedded in html

        $dom = new \DOMDocument();
        @$dom->loadHTML(mb_convert_encoding($html, 'HTML-ENTITIES', 'UTF-8'));
        $images = $dom->getElementsByTagName('img');
        if(!$images->length) return; // not needed?

        $assetsPath = $this->pages->get($page->id)->filesManager()->url();

        $field_image = $this->wire('fields')->get($img_field);
        $allowedExts = $field_image ? explode(' ', strtolower($field_image->extensions)) : [];

        $extCount = 0;
        $errors = [];
        foreach ($images as $image) {
            $img_url = $image->getAttribute('src');
            if(!filter_var($img_url, FILTER_VALIDATE_URL)) continue;

            // Skip non-absolute or already-local images
            $parsed  = parse_url($img_url);
            $imgPath = $parsed['path'] ?? '';
            $imgExt  = strtolower(pathinfo($imgPath, PATHINFO_EXTENSION));

            // skip images with no detectable extension — they trigger Sanitizer warnings in PW core
            if(!$imgExt) {
                $errors[] = "Skipped (no extension): $img_url";
                continue;
            }

            if($allowedExts && !in_array($imgExt, $allowedExts)) {
                $errors[] = "Extension .$imgExt not permitted for $img_url";
                continue;
            }

            try {
                $page->$img_field->add($img_url);
            } catch(\Throwable $e) {
                $errors[] = "Could not add image $img_url: " . $e->getMessage();
                continue;
            }

            // Confirm the add succeeded before touching ->last()
            $addedImg = $page->$img_field->last();
            if(!$addedImg) continue;

	        if($image->getAttribute('title') != ''){
	            $addedImg->description = $image->getAttribute('title');
	        }
	        elseif($image->getAttribute('alt') != '' && $image->getAttribute('alt') != basename($image->getAttribute('src'))){
	            $addedImg->description = $image->getAttribute('alt');
			}

            //resize image to make version to match the size originally in the RTE
            //check to make sure size is different to downloaded version before resizing
            $imgWidth = $image->getAttribute('width');
			if(strpos($imgWidth,'%') !== false) $imgWidth = ''; // don't resize if image is % size.
			if($imgWidth && $imgWidth != $addedImg->width) {
                $imgForRte = $addedImg->size($imgWidth, 0);
            }
            else {
                $imgForRte = $addedImg;
            }
			$image->setAttribute('src', $imgForRte->url);

			if(!$dont_wrap_figure) {
				$figure = $dom->createElement('figure');
				$image->parentNode->replaceChild($figure, $image);
				$figure->appendChild($image);
				if($image->getAttribute('alt') != '' && $image->getAttribute('alt') != basename($image->getAttribute('src'))) {
					$figcaption = $dom->createElement('figcaption');
					$figcaption->nodeValue = $image->getAttribute('alt');
					$figure->appendChild($figcaption);
				}
			}

            $extCount++;
        }

        if(!$extCount) return;
    	$page->of(false);

   		$page->$ta_field = preg_replace('/^<!DOCTYPE.+?>/', '', str_replace( array('<html>', '</html>', '<body>', '</body>'), array('', '', '', ''), $dom->saveHTML()));

   	 	$page->save($ta_field);
   	 	$this->message("image links updated to local images.");

        $page->save($img_field);
        $this->message("external images imported to page");


    }



    // public function ___upgrade($fromVersion, $toVersion) {

    //     // change module config settings for the stored templates from name to ID
    //     if(isset($this->data['enabledTemplates'])) {

	// 		foreach(json_decode($this->data['enabledTemplates']) as $tplName) {

    //         }

	// 		$configData = $this->wire('modules')->getModuleConfigData("ImportExternalImages");
    //     	unset($configData['enabledTemplates']);
    //        $this->wire('modules')->saveModuleConfigData($this, $configData);

    //     }

    // }



} // end class