Merge branch '2048-improve-messages-config-exceptions' of…

framework/helpers/BaseFileHelper.php

@@ -10,6 +10,7 @@
 namespace yii\helpers;
 
 use Yii;
+use yii\base\InvalidParamException;
 
 /**
  * BaseFileHelper provides concrete implementation for [[FileHelper]].
@@ -22,6 +23,11 @@ use Yii;
  */
 class BaseFileHelper
 {
+	const PATTERN_NODIR = 1;
+	const PATTERN_ENDSWITH = 4;
+	const PATTERN_MUSTBEDIR = 8;
+	const PATTERN_NEGATIVE = 16;
+
 	/**
 	 * Normalizes a file/directory path.
 	 * After normalization, the directory separators in the path will be `DIRECTORY_SEPARATOR`,
@@ -242,22 +248,47 @@ class BaseFileHelper
 	 *   * false: the directory or file will NOT be returned (the "only" and "except" options will be ignored)
 	 *   * null: the "only" and "except" options will determine whether the directory or file should be returned
 	 *
-	 * - only: array, list of patterns that the file paths should match if they want to be returned.
-	 *   A path matches a pattern if it contains the pattern string at its end.
-	 *   For example, '.php' matches all file paths ending with '.php'.
-	 *   Note, the '/' characters in a pattern matches both '/' and '\' in the paths.
-	 *   If a file path matches a pattern in both "only" and "except", it will NOT be returned.
-	 * - except: array, list of patterns that the file paths or directory paths should match if they want to be excluded from the result.
-	 *   A path matches a pattern if it contains the pattern string at its end.
+	 * - except: array, list of patterns excluding from the results matching file or directory paths.
 	 *   Patterns ending with '/' apply to directory paths only, and patterns not ending with '/'
 	 *   apply to file paths only. For example, '/a/b' matches all file paths ending with '/a/b';
-	 *   and '.svn/' matches directory paths ending with '.svn'. Note, the '/' characters in a pattern matches
-	 *   both '/' and '\' in the paths.
+	 *   and '.svn/' matches directory paths ending with '.svn'.
+	 *   If the pattern does not contain a slash /, it is treated as a shell glob pattern and checked for a match against the pathname relative to $dir.
+	 *   Otherwise, the pattern is treated as a shell glob suitable for consumption by fnmatch(3) with the FNM_PATHNAME flag: wildcards in the pattern will not match a / in the pathname.
+	 *   For example, "views/*.php" matches "views/index.php" but not "views/controller/index.php".
+	 *   A leading slash matches the beginning of the pathname. For example, "/*.php" matches "index.php" but not "views/start/index.php".
+	 *   An optional prefix "!" which negates the pattern; any matching file excluded by a previous pattern will become included again.
+	 *   If a negated pattern matches, this will override lower precedence patterns sources. Put a backslash ("\") in front of the first "!"
+	 *   for patterns that begin with a literal "!", for example, "\!important!.txt".
+	 *   Note, the '/' characters in a pattern matches both '/' and '\' in the paths.
+	 * - only: array, list of patterns that the file paths should match if they are to be returned. Directory paths are not checked against them.
+	 *   Same pattern matching rules as in the "except" option are used.
+	 *   If a file path matches a pattern in both "only" and "except", it will NOT be returned.
 	 * - recursive: boolean, whether the files under the subdirectories should also be looked for. Defaults to true.
 	 * @return array files found under the directory. The file list is sorted.
+	 * @throws InvalidParamException if the dir is invalid.
 	 */
 	public static function findFiles($dir, $options = [])
 	{
+		if (!is_dir($dir)) {
+			throw new InvalidParamException('The dir argument must be a directory.');
+		}
+		$dir = rtrim($dir, DIRECTORY_SEPARATOR);
+		if (!isset($options['basePath'])) {
+			$options['basePath'] = realpath($dir);
+			// this should also be done only once
+			if (isset($options['except'])) {
+				foreach($options['except'] as $key=>$value) {
+					if (is_string($value))
+						$options['except'][$key] = static::parseExcludePattern($value);
+				}
+			}
+			if (isset($options['only'])) {
+				foreach($options['only'] as $key=>$value) {
+					if (is_string($value))
+						$options['only'][$key] = static::parseExcludePattern($value);
+				}
+			}
+		}
 		$list = [];
 		$handle = opendir($dir);
 		while (($file = readdir($handle)) !== false) {
@@ -298,25 +329,18 @@ class BaseFileHelper
 		}
 
 		$path = str_replace('\\', '/', $path);
-		if ($isDir = is_dir($path)) {
-			$path .= '/';
-		}
-		$n = StringHelper::byteLength($path);
 
 		if (!empty($options['except'])) {
-			foreach ($options['except'] as $name) {
-				if (StringHelper::byteSubstr($path, -StringHelper::byteLength($name), $n) === $name) {
-					return false;
-				}
+			if (($except = self::lastExcludeMatchingFromList($options['basePath'], $path, $options['except'])) !== null) {
+				return $except['flags'] & self::PATTERN_NEGATIVE;
 			}
 		}
 
-		if (!$isDir && !empty($options['only'])) {
-			foreach ($options['only'] as $name) {
-				if (StringHelper::byteSubstr($path, -StringHelper::byteLength($name), $n) === $name) {
+		if (!is_dir($path) && !empty($options['only'])) {
+			if (($except = self::lastExcludeMatchingFromList($options['basePath'], $path, $options['only'])) !== null) {
+				// don't check PATTERN_NEGATIVE since those entries are not prefixed with !
 				return true;
 			}
-			}
 			return false;
 		}
 		return true;
@@ -347,4 +371,174 @@ class BaseFileHelper
 		chmod($path, $mode);
 		return $result;
 	}
+
+	/**
+	 * Performs a simple comparison of file or directory names.
+	 *
+	 * Based on match_basename() from dir.c of git 1.8.5.3 sources.
+	 *
+	 * @param string $baseName file or directory name to compare with the pattern
+	 * @param string $pattern the pattern that $baseName will be compared against
+	 * @param integer|boolean $firstWildcard location of first wildcard character in the $pattern
+	 * @param integer $flags pattern flags
+	 * @return boolean wheter the name matches against pattern
+	 */
+	private static function matchBasename($baseName, $pattern, $firstWildcard, $flags)
+	{
+		if ($firstWildcard === false) {
+			if ($pattern === $baseName) {
+				return true;
+			}
+		} else if ($flags & self::PATTERN_ENDSWITH) {
+			/* "*literal" matching against "fooliteral" */
+			$n = StringHelper::byteLength($pattern);
+			if (StringHelper::byteSubstr($pattern, 1, $n) === StringHelper::byteSubstr($baseName, -$n, $n)) {
+				return true;
+			}
+		}
+		return fnmatch($pattern, $baseName, 0);
+	}
+
+	/**
+	 * Compares a path part against a pattern with optional wildcards.
+	 *
+	 * Based on match_pathname() from dir.c of git 1.8.5.3 sources.
+	 *
+	 * @param string $path full path to compare
+	 * @param string $basePath base of path that will not be compared
+	 * @param string $pattern the pattern that path part will be compared against
+	 * @param integer|boolean $firstWildcard location of first wildcard character in the $pattern
+	 * @param integer $flags pattern flags
+	 * @return boolean wheter the path part matches against pattern
+	 */
+	private static function matchPathname($path, $basePath, $pattern, $firstWildcard, $flags)
+	{
+		// match with FNM_PATHNAME; the pattern has base implicitly in front of it.
+		if (isset($pattern[0]) && $pattern[0] == '/') {
+			$pattern = StringHelper::byteSubstr($pattern, 1, StringHelper::byteLength($pattern));
+			if ($firstWildcard !== false && $firstWildcard !== 0) {
+				$firstWildcard--;
+			}
+		}
+
+		$namelen = StringHelper::byteLength($path) - (empty($basePath) ? 0 : StringHelper::byteLength($basePath) + 1);
+		$name = StringHelper::byteSubstr($path, -$namelen, $namelen);
+
+		if ($firstWildcard !== 0) {
+			if ($firstWildcard === false) {
+				$firstWildcard = StringHelper::byteLength($pattern);
+			}
+			// if the non-wildcard part is longer than the remaining pathname, surely it cannot match.
+			if ($firstWildcard > $namelen) {
+				return false;
+			}
+
+			if (strncmp($pattern, $name, $firstWildcard)) {
+				return false;
+			}
+			$pattern = StringHelper::byteSubstr($pattern, $firstWildcard, StringHelper::byteLength($pattern));
+			$name = StringHelper::byteSubstr($name, $firstWildcard, $namelen);
+
+			// If the whole pattern did not have a wildcard, then our prefix match is all we need; we do not need to call fnmatch at all.
+			if (empty($pattern) && empty($name)) {
+				return true;
+			}
+		}
+
+		return fnmatch($pattern, $name, FNM_PATHNAME);
+	}
+
+	/**
+	 * Scan the given exclude list in reverse to see whether pathname
+	 * should be ignored.  The first match (i.e. the last on the list), if
+	 * any, determines the fate.  Returns the element which
+	 * matched, or null for undecided.
+	 *
+	 * Based on last_exclude_matching_from_list() from dir.c of git 1.8.5.3 sources.
+	 *
+	 * @param string $basePath
+	 * @param string $path
+	 * @param array $excludes list of patterns to match $path against
+	 * @return string null or one of $excludes item as an array with keys: 'pattern', 'flags'
+	 * @throws InvalidParamException if any of the exclude patterns is not a string or an array with keys: pattern, flags, firstWildcard.
+	 */
+	private static function lastExcludeMatchingFromList($basePath, $path, $excludes)
+	{
+		foreach(array_reverse($excludes) as $exclude) {
+			if (is_string($exclude)) {
+				$exclude = self::parseExcludePattern($exclude);
+			}
+			if (!isset($exclude['pattern']) || !isset($exclude['flags']) || !isset($exclude['firstWildcard'])) {
+				throw new InvalidParamException('If exclude/include pattern is an array it must contain the pattern, flags and firstWildcard keys.');
+			}
+			if ($exclude['flags'] & self::PATTERN_MUSTBEDIR && !is_dir($path)) {
+				continue;
+			}
+
+			if ($exclude['flags'] & self::PATTERN_NODIR) {
+				if (self::matchBasename(basename($path), $exclude['pattern'], $exclude['firstWildcard'], $exclude['flags'])) {
+					return $exclude;
+				}
+				continue;
+			}
+
+			if (self::matchPathname($path, $basePath, $exclude['pattern'], $exclude['firstWildcard'], $exclude['flags'])) {
+				return $exclude;
+			}
+		}
+		return null;
+	}
+
+	/**
+	 * Processes the pattern, stripping special characters like / and ! from the beginning and settings flags instead.
+	 * @param string $pattern
+	 * @return array with keys: (string)pattern, (int)flags, (int|boolean)firstWildcard
+	 * @throws InvalidParamException if the pattern is not a string.
+	 */
+	private static function parseExcludePattern($pattern)
+	{
+		if (!is_string($pattern)) {
+			throw new InvalidParamException('Exclude/include pattern must be a string.');
+		}
+		$result = array(
+			'pattern' => $pattern,
+			'flags' => 0,
+			'firstWildcard' => false,
+		);
+		if (!isset($pattern[0]))
+			return $result;
+
+		if ($pattern[0] == '!') {
+			$result['flags'] |= self::PATTERN_NEGATIVE;
+			$pattern = StringHelper::byteSubstr($pattern, 1, StringHelper::byteLength($pattern));
+		}
+		$len = StringHelper::byteLength($pattern);
+		if ($len && StringHelper::byteSubstr($pattern, -1, 1) == '/') {
+			$pattern = StringHelper::byteSubstr($pattern, 0, -1);
+			$len--;
+			$result['flags'] |= self::PATTERN_MUSTBEDIR;
+		}
+		if (strpos($pattern, '/') === false)
+			$result['flags'] |= self::PATTERN_NODIR;
+		$result['firstWildcard'] = self::firstWildcardInPattern($pattern);
+		if ($pattern[0] == '*' && self::firstWildcardInPattern(StringHelper::byteSubstr($pattern, 1, StringHelper::byteLength($pattern))) === false)
+			$result['flags'] |= self::PATTERN_ENDSWITH;
+		$result['pattern'] = $pattern;
+		return $result;
+	}
+
+	/**
+	 * Searches for the first wildcard character in the pattern.
+	 * @param string $pattern the pattern to search in
+	 * @return integer|boolean position of first wildcard character or false if not found
+	 */
+	private static function firstWildcardInPattern($pattern)
+	{
+		$wildcards = array('*','?','[','\\');
+		$wildcardSearch = function($r, $c) use ($pattern) {
+			$p = strpos($pattern, $c);
+			return $r===false ? $p : ($p===false ? $r : min($r, $p));
+		};
+		return array_reduce($wildcards, $wildcardSearch, false);
+	}
 }

framework/messages/config.php

@@ -22,17 +22,14 @@ return [
 	// boolean, whether to remove messages that no longer appear in the source code.
 	// Defaults to false, which means each of these messages will be enclosed with a pair of '@@' marks.
 	'removeUnused' => false,
-	// array, list of patterns that specify which files/directories should be processed.
+	// array, list of patterns that specify which files/directories should NOT be processed.
 	// If empty or not set, all files/directories will be processed.
 	// A path matches a pattern if it contains the pattern string at its end. For example,
 	// '/a/b' will match all files and directories ending with '/a/b';
-	// and the '.svn' will match all files and directories whose name ends with '.svn'.
+	// the '*.svn' will match all files and directories whose name ends with '.svn'.
+	// and the '.svn' will match all files and directories named exactly '.svn'.
 	// Note, the '/' characters in a pattern matches both '/' and '\'.
-	// If a file/directory matches both a pattern in "only" and "except", it will NOT be processed.
-	'only' => ['.php'],
-	// array, list of patterns that specify which files/directories should NOT be processed.
-	// If empty or not set, all files/directories will be processed.
-	// Please refer to "only" for details about the patterns.
+	// See helpers/FileHelper::findFiles() description for more details on pattern matching rules.
 	'except' => [
 		'.svn',
 		'.git',
@@ -42,8 +39,13 @@ return [
 		'.hgkeep',
 		'/messages',
 	],
-	// Generated file format. Can be either "php", "po" or "db".
+	// array, list of patterns that specify which files (not directories) should be processed.
+	// If empty or not set, all files will be processed.
+	// Please refer to "except" for details about the patterns.
+	// If a file/directory matches both a pattern in "only" and "except", it will NOT be processed.
+	'only' => ['*.php'],
+	// Generated file format. Can be "php", "db" or "po".
 	'format' => 'php',
-	// Connection component ID for "db".
-	//'connectionID' => 'db',
+	// Connection component ID for "db" format.
+	//'db' => 'db',
 ];

framework/views/messageConfig.php

@@ -22,17 +22,19 @@ return [
 	// boolean, whether to remove messages that no longer appear in the source code.
 	// Defaults to false, which means each of these messages will be enclosed with a pair of '@@' marks.
 	'removeUnused' => false,
-	// array, list of patterns that specify which files/directories should be processed.
+	// array, list of patterns that specify which files/directories should NOT be processed.
 	// If empty or not set, all files/directories will be processed.
 	// A path matches a pattern if it contains the pattern string at its end. For example,
 	// '/a/b' will match all files and directories ending with '/a/b';
-	// and the '.svn' will match all files and directories whose name ends with '.svn'.
+	// the '*.svn' will match all files and directories whose name ends with '.svn'.
+	// and the '.svn' will match all files and directories named exactly '.svn'.
 	// Note, the '/' characters in a pattern matches both '/' and '\'.
-	// If a file/directory matches both a pattern in "only" and "except", it will NOT be processed.
+	// See helpers/FileHelper::findFiles() description for more details on pattern matching rules.
 	'only' => ['.php'],
-	// array, list of patterns that specify which files/directories should NOT be processed.
-	// If empty or not set, all files/directories will be processed.
-	// Please refer to "only" for details about the patterns.
+	// array, list of patterns that specify which files (not directories) should be processed.
+	// If empty or not set, all files will be processed.
+	// Please refer to "except" for details about the patterns.
+	// If a file/directory matches both a pattern in "only" and "except", it will NOT be processed.
 	'except' => [
 		'.svn',
 		'.git',

tests/unit/framework/helpers/FileHelperTest.php

@@ -253,23 +253,58 @@ class FileHelperTest extends TestCase
 	 */
 	public function testFindFilesExclude()
 	{
-		$dirName = 'test_dir';
-		$fileName = 'test_file.txt';
-		$excludeFileName = 'exclude_file.txt';
-		$this->createFileStructure([
-			$dirName => [
-				$fileName => 'file content',
-				$excludeFileName => 'exclude file content',
-			],
-		]);
-		$basePath = $this->testFilePath;
-		$dirName = $basePath . DIRECTORY_SEPARATOR . $dirName;
+		$basePath = $this->testFilePath . DIRECTORY_SEPARATOR;
+		$dirs = array('', 'one', 'one'.DIRECTORY_SEPARATOR.'two', 'three');
+		$files = array_fill_keys(array_map(function($n){return "a.$n";}, range(1,8)), 'file contents');
+
+		$tree = $files;
+		$root = $files;
+		$flat = array();
+		foreach($dirs as $dir) {
+			foreach($files as $fileName=>$contents) {
+				$flat[] = rtrim($basePath.$dir,DIRECTORY_SEPARATOR).DIRECTORY_SEPARATOR.$fileName;
+			}
+			if ($dir === '') continue;
+			$parts = explode(DIRECTORY_SEPARATOR, $dir);
+			$last = array_pop($parts);
+			$parent = array_pop($parts);
+			$tree[$last] = $files;
+			if ($parent !== null) {
+				$tree[$parent][$last] = &$tree[$last];
+			} else {
+				$root[$last] = &$tree[$last];
+			}
+		}
+		$this->createFileStructure($root);
 
-		$options = [
-			'except' => [$excludeFileName],
-		];
-		$foundFiles = FileHelper::findFiles($dirName, $options);
-		$this->assertEquals([$dirName . DIRECTORY_SEPARATOR . $fileName], $foundFiles);
+		// range
+		$foundFiles = FileHelper::findFiles($basePath, ['except' => ['a.[2-8]']]);
+		sort($foundFiles);
+		$expect = array_values(array_filter($flat, function($p){return substr($p, -3)==='a.1';}));
+		$this->assertEquals($expect, $foundFiles);
+
+		// suffix
+		$foundFiles = FileHelper::findFiles($basePath, ['except' => ['*.1']]);
+		sort($foundFiles);
+		$expect = array_values(array_filter($flat, function($p){return substr($p, -3)!=='a.1';}));
+		$this->assertEquals($expect, $foundFiles);
+
+		// dir
+		$foundFiles = FileHelper::findFiles($basePath, ['except' => ['/one']]);
+		sort($foundFiles);
+		$expect = array_values(array_filter($flat, function($p){return strpos($p, DIRECTORY_SEPARATOR.'one')===false;}));
+		$this->assertEquals($expect, $foundFiles);
+
+		// dir contents
+		$foundFiles = FileHelper::findFiles($basePath, ['except' => ['?*/a.1']]);
+		sort($foundFiles);
+		$expect = array_values(array_filter($flat, function($p){
+			return substr($p, -11, 10)==='one'.DIRECTORY_SEPARATOR.'two'.DIRECTORY_SEPARATOR.'a.' || (
+				substr($p, -8)!==DIRECTORY_SEPARATOR.'one'.DIRECTORY_SEPARATOR.'a.1' &&
+				substr($p, -10)!==DIRECTORY_SEPARATOR.'three'.DIRECTORY_SEPARATOR.'a.1'
+			);
+		}));
+		$this->assertEquals($expect, $foundFiles);
 	}
 
 	public function testCreateDirectory()