    001/*
    002 * Licensed to the Apache Software Foundation (ASF) under one or more
    003 * contributor license agreements.  See the NOTICE file distributed with
    004 * this work for additional information regarding copyright ownership.
    005 * The ASF licenses this file to You under the Apache License, Version 2.0
    006 * (the "License"); you may not use this file except in compliance with
    007 * the License.  You may obtain a copy of the License at
    008 *
    009 *      https://www.apache.org/licenses/LICENSE-2.0
    010 *
    011 * Unless required by applicable law or agreed to in writing, software
    012 * distributed under the License is distributed on an "AS IS" BASIS,
    013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    014 * See the License for the specific language governing permissions and
    015 * limitations under the License.
    016 */
    017package org.apache.commons.validator.routines;
    018
    019import java.io.Serializable;
    020import java.io.UnsupportedEncodingException;
    021import java.net.URI;
    022import java.net.URISyntaxException;
    023import java.net.URLDecoder;
    024import java.nio.charset.StandardCharsets;
    025import java.util.Collections;
    026import java.util.HashSet;
    027import java.util.Locale;
    028import java.util.Set;
    029import java.util.regex.Matcher;
    030import java.util.regex.Pattern;
    031
    032import org.apache.commons.validator.GenericValidator;
    033
    034/**
    035 * <p><strong>URL Validation</strong> routines.</p>
    036 * Behavior of validation is modified by passing in options:
    037 * <ul>
    038 * <li>ALLOW_2_SLASHES - [FALSE]  Allows double '/' characters in the path
    039 * component.</li>
    040 * <li>NO_FRAGMENT- [FALSE]  By default fragments are allowed, if this option is
    041 * included then fragments are flagged as illegal.</li>
    042 * <li>ALLOW_ALL_SCHEMES - [FALSE] By default only http, https, and ftp are
    043 * considered valid schemes.  Enabling this option will let any scheme pass validation.</li>
    044 * </ul>
    045 *
    046 * <p>Originally based in on php script by Debbie Dyer, validation.php v1.2b, Date: 03/07/02,
    047 * https://javascript.internet.com. However, this validation now bears little resemblance
    048 * to the php original.</p>
    049 * <pre>
    050 *   Example of usage:
    051 *   Construct a UrlValidator with valid schemes of "http", and "https".
    052 *
    053 *    String[] schemes = {"http","https"}.
    054 *    UrlValidator urlValidator = new UrlValidator(schemes);
    055 *    if (urlValidator.isValid("ftp://foo.bar.com/")) {
    056 *       System.out.println("URL is valid");
    057 *    } else {
    058 *       System.out.println("URL is invalid");
    059 *    }
    060 *
    061 *    prints "URL is invalid"
    062 *   If instead the default constructor is used.
    063 *
    064 *    UrlValidator urlValidator = new UrlValidator();
    065 *    if (urlValidator.isValid("ftp://foo.bar.com/")) {
    066 *       System.out.println("URL is valid");
    067 *    } else {
    068 *       System.out.println("URL is invalid");
    069 *    }
    070 *
    071 *   prints out "URL is valid"
    072 *  </pre>
    073 *
    074 * @see
    075 * <a href="https://www.ietf.org/rfc/rfc2396.txt">
    076 *  Uniform Resource Identifiers (URI): Generic Syntax
    077 * </a>
    078 *
    079 * @since 1.4
    080 */
    081public class UrlValidator implements Serializable {
    082
    083    private static final long serialVersionUID = 7557161713937335013L;
    084
    085    private static final int MAX_UNSIGNED_16_BIT_INT = 0xFFFF; // port max
    086
    087    /**
    088     * Allows all validly formatted schemes to pass validation instead of
    089     * supplying a set of valid schemes.
    090     */
    091    public static final long ALLOW_ALL_SCHEMES = 1 << 0;
    092
    093    /**
    094     * Allow two slashes in the path component of the URL.
    095     */
    096    public static final long ALLOW_2_SLASHES = 1 << 1;
    097
    098    /**
    099     * Enabling this options disallows any URL fragments.
    100     */
    101    public static final long NO_FRAGMENTS = 1 << 2;
    102
    103    /**
    104     * Allow local URLs, such as https://localhost/ or https://machine/ .
    105     * This enables a broad-brush check, for complex local machine name
    106     *  validation requirements you should create your validator with
    107     *  a {@link RegexValidator} instead ({@link #UrlValidator(RegexValidator, long)})
    108     */
    109    public static final long ALLOW_LOCAL_URLS = 1 << 3; // CHECKSTYLE IGNORE MagicNumber
    110
    111    /**
    112     * Protocol scheme (for example, http, ftp, https).
    113     */
    114    private static final String SCHEME_REGEX = "^\\p{Alpha}[\\p{Alnum}\\+\\-\\.]*";
    115    private static final Pattern SCHEME_PATTERN = Pattern.compile(SCHEME_REGEX);
    116
    117    // Drop numeric, and  "+-." for now
    118    // TODO does not allow for optional userinfo.
    119    // Validation of character set is done by isValidAuthority
    120    private static final String AUTHORITY_CHARS_REGEX = "\\p{Alnum}\\-\\."; // allows for IPV4 but not IPV6
    121    // Captured inside [ ] in AUTHORITY_REGEX and validated by InetAddressValidator.isValidInet6Address, so the
    122    // dot is allowed for IPv4-mapped/embedded forms (for example ::ffff:1.2.3.4 or 2001:db8::1.2.3.4), not just ::FFFF:
    123    private static final String IPV6_REGEX = "[0-9a-fA-F:.]+"; // the brackets remove the port-prefix ':' ambiguity
    124
    125    // userinfo    = *( unreserved / pct-encoded / sub-delims / ":" )
    126    // unreserved    = ALPHA / DIGIT / "-" / "." / "_" / "~"
    127    // sub-delims    = "!" / "$" / "&" / "'" / "(" / ")" / "*" / "+" / "," / ";" / "="
    128    // We assume that password has the same valid chars as user info
    129    private static final String USERINFO_CHARS_REGEX = "[a-zA-Z0-9%-._~!$&'()*+,;=]";
    130
    131    // since neither ':' nor '@' are allowed chars, we don't need to use non-greedy matching
    132    private static final String USERINFO_FIELD_REGEX =
    133            USERINFO_CHARS_REGEX + "+" + // At least one character for the name
    134            "(?::" + USERINFO_CHARS_REGEX + "*)?@"; // colon and password may be absent
    135
    136    // Optional userinfo ("user:pass@") precedes the host so it may come before either host form: a bracketed IPv6
    137    // literal (group 1) or a hostname/IPv4 host (group 2). Group 3 is the port, group 4 any trailing remainder.
    138    private static final String AUTHORITY_REGEX =
    139            "(?:" + USERINFO_FIELD_REGEX + ")?(?:\\[(" + IPV6_REGEX + ")\\]|([" + AUTHORITY_CHARS_REGEX + "]*))(?::(\\d*))?(.*)?";
    140    private static final Pattern AUTHORITY_PATTERN = Pattern.compile(AUTHORITY_REGEX);
    141
    142    private static final int PARSE_AUTHORITY_IPV6 = 1;
    143
    144    private static final int PARSE_AUTHORITY_HOST_IP = 2; // excludes userinfo, if present
    145
    146    private static final int PARSE_AUTHORITY_PORT = 3; // excludes leading colon
    147
    148    /**
    149     * Should always be empty. The code currently allows spaces.
    150     */
    151    private static final int PARSE_AUTHORITY_EXTRA = 4;
    152
    153    private static final String PATH_REGEX = "^(/[-\\w:@&?=+,.!/~*'%$_;\\(\\)]*)?$";
    154    private static final Pattern PATH_PATTERN = Pattern.compile(PATH_REGEX);
    155
    156    private static final String QUERY_REGEX = "^(\\S*)$";
    157    private static final Pattern QUERY_PATTERN = Pattern.compile(QUERY_REGEX);
    158
    159    /**
    160     * If no schemes are provided, default to this set.
    161     */
    162    private static final String[] DEFAULT_SCHEMES = {"http", "https", "ftp"}; // Must be lower-case
    163
    164    /**
    165     * Singleton instance of this class with default schemes and options.
    166     */
    167    private static final UrlValidator DEFAULT_URL_VALIDATOR = new UrlValidator();
    168
    169    /**
    170     * Returns the singleton instance of this class with default schemes and options.
    171     *
    172     * @return singleton instance with default schemes and options
    173     */
    174    public static UrlValidator getInstance() {
    175        return DEFAULT_URL_VALIDATOR;
    176    }
    177
    178    /**
    179     * Tests whether the given flag is on. If the flag is not a power of 2
    180     * (for example, 3) this tests whether the combination of flags is on.
    181     *
    182     * @param flag Flag value to check.
    183     * @param options what to check
    184     * @return whether the specified flag value is on.
    185     */
    186    private static boolean isOn(final long flag, final long options) {
    187        return (options & flag) > 0;
    188    }
    189
    190    /**
    191     * Holds the set of current validation options.
    192     */
    193    private final long options;
    194
    195    /**
    196     * The set of schemes that are allowed to be in a URL.
    197     */
    198    private final Set<String> allowedSchemes; // Must be lower-case
    199
    200    /**
    201     * Regular expressions used to manually validate authorities if IANA
    202     * domain name validation isn't desired.
    203     */
    204    private final RegexValidator authorityValidator;
    205
    206    /**
    207     * The domain validator.
    208     */
    209    private final DomainValidator domainValidator;
    210
    211    /**
    212     * Constructs a new instance with default properties.
    213     */
    214    public UrlValidator() {
    215        this(null);
    216    }
    217
    218    /**
    219     * Constructs a new instance with the given validation options.
    220     *
    221     * @param options The options should be set using the public constants declared in
    222     * this class.  To set multiple options you simply add them together.  For example,
    223     * ALLOW_2_SLASHES + NO_FRAGMENTS enables both of those options.
    224     */
    225    public UrlValidator(final long options) {
    226        this(null, null, options);
    227    }
    228
    229    /**
    230     * Constructs a new instance with the given validation options.
    231     *
    232     * @param authorityValidator Regular expression validator used to validate the authority part
    233     * This allows the user to override the standard set of domains.
    234     * @param options Validation options. Set using the public constants of this class.
    235     * To set multiple options, simply add them together:
    236     * <p>{@code ALLOW_2_SLASHES + NO_FRAGMENTS}</p>
    237     * enables both of those options.
    238     */
    239    public UrlValidator(final RegexValidator authorityValidator, final long options) {
    240        this(null, authorityValidator, options);
    241    }
    242
    243    /**
    244     * Behavior of validation is modified by passing in several strings options:
    245     *
    246     * @param schemes Pass in one or more URL schemes to consider valid, passing in
    247     *        a null will default to "http,https,ftp" being valid.
    248     *        If a non-null schemes is specified then all valid schemes must
    249     *        be specified. Setting the ALLOW_ALL_SCHEMES option will
    250     *        ignore the contents of schemes.
    251     */
    252    public UrlValidator(final String[] schemes) {
    253        this(schemes, 0L);
    254    }
    255
    256    /**
    257     * Behavior of validation is modified by passing in options:
    258     *
    259     * @param schemes The set of valid schemes. Ignored if the ALLOW_ALL_SCHEMES option is set.
    260     * @param options The options should be set using the public constants declared in
    261     * this class.  To set multiple options you simply add them together.  For example,
    262     * ALLOW_2_SLASHES + NO_FRAGMENTS enables both of those options.
    263     */
    264    public UrlValidator(final String[] schemes, final long options) {
    265        this(schemes, null, options);
    266    }
    267
    268    /**
    269     * Customizable constructor. Validation behavior is modified by passing in options.
    270     *
    271     * @param schemes The set of valid schemes. Ignored if the ALLOW_ALL_SCHEMES option is set.
    272     * @param authorityValidator Regular expression validator used to validate the authority part
    273     * @param options Validation options. Set using the public constants of this class.
    274     * To set multiple options, simply add them together:
    275     * <p>{@code ALLOW_2_SLASHES + NO_FRAGMENTS}</p>
    276     * enables both of those options.
    277     */
    278    public UrlValidator(final String[] schemes, final RegexValidator authorityValidator, final long options) {
    279        this(schemes, authorityValidator, options, DomainValidator.getInstance(isOn(ALLOW_LOCAL_URLS, options)));
    280    }
    281
    282    /**
    283     * Customizable constructor. Validation behavior is modified by passing in options.
    284     *
    285     * @param schemes The set of valid schemes. Ignored if the ALLOW_ALL_SCHEMES option is set.
    286     * @param authorityValidator Regular expression validator used to validate the authority part
    287     * @param options Validation options. Set using the public constants of this class.
    288     * To set multiple options, simply add them together:
    289     * <p>{@code ALLOW_2_SLASHES + NO_FRAGMENTS}</p>
    290     * enables both of those options.
    291     * @param domainValidator The DomainValidator to use; must agree with ALLOW_LOCAL_URLS setting
    292     * @since 1.7
    293     */
    294    public UrlValidator(String[] schemes, final RegexValidator authorityValidator, final long options, final DomainValidator domainValidator) {
    295        this.options = options;
    296        if (domainValidator == null) {
    297            throw new IllegalArgumentException("DomainValidator must not be null");
    298        }
    299        if (domainValidator.isAllowLocal() != (options & ALLOW_LOCAL_URLS) > 0) {
    300            throw new IllegalArgumentException("DomainValidator disagrees with ALLOW_LOCAL_URLS setting");
    301        }
    302        this.domainValidator = domainValidator;
    303
    304        if (isOn(ALLOW_ALL_SCHEMES)) {
    305            allowedSchemes = Collections.emptySet();
    306        } else {
    307            if (schemes == null) {
    308                schemes = DEFAULT_SCHEMES;
    309            }
    310            allowedSchemes = new HashSet<>(schemes.length);
    311            for (final String scheme : schemes) {
    312                allowedSchemes.add(scheme.toLowerCase(Locale.ENGLISH));
    313            }
    314        }
    315
    316        this.authorityValidator = authorityValidator;
    317    }
    318
    319    /**
    320     * Returns the number of times the token appears in the target.
    321     *
    322     * @param token Token value to be counted.
    323     * @param target Target value to count tokens in.
    324     * @return The number of tokens.
    325     */
    326    protected int countToken(final String token, final String target) {
    327        int tokenIndex = 0;
    328        int count = 0;
    329        while (tokenIndex != -1) {
    330            tokenIndex = target.indexOf(token, tokenIndex);
    331            if (tokenIndex > -1) {
    332                tokenIndex++;
    333                count++;
    334            }
    335        }
    336        return count;
    337    }
    338
    339    /**
    340     * Tests whether the given flag is off.  If the flag is not a power of 2
    341     * (for example, 3) this tests whether the combination of flags is off.
    342     *
    343     * @param flag Flag value to check.
    344     * @return whether the specified flag value is off.
    345     */
    346    private boolean isOff(final long flag) {
    347        return (options & flag) == 0;
    348    }
    349
    350    /**
    351     * Tests whether the given flag is on.  If the flag is not a power of 2
    352     * (for example, 3) this tests whether the combination of flags is on.
    353     *
    354     * @param flag Flag value to check.
    355     * @return whether the specified flag value is on.
    356     */
    357    private boolean isOn(final long flag) {
    358        return (options & flag) > 0;
    359    }
    360
    361    /**
    362     * <p>Checks if a field has a valid URL address.</p>
    363     *
    364     * Note that the method calls #isValidAuthority()
    365     * which checks that the domain is valid.
    366     *
    367     * @param value The value validation is being performed on.  A {@code null}
    368     * value is considered invalid.
    369     * @return true if the URL is valid.
    370     */
    371    public boolean isValid(final String value) {
    372        if (value == null) {
    373            return false;
    374        }
    375        final URI uri; // ensure value is a valid URI
    376        try {
    377            uri = new URI(value);
    378        } catch (final URISyntaxException e) {
    379            return false;
    380        }
    381        // OK, perform additional validation
    382        final String scheme = uri.getScheme();
    383        if (!isValidScheme(scheme)) {
    384            return false;
    385        }
    386        final String authority = uri.getRawAuthority();
    387        if ("file".equals(scheme) && GenericValidator.isBlankOrNull(authority)) { // Special case - file: allows an empty authority
    388            return true; // this is a local file - nothing more to do here
    389        }
    390        // Validate the authority
    391        if ("file".equals(scheme) && authority != null && authority.contains(":") || !isValidAuthority(authority)) {
    392            return false;
    393        }
    394        if (!isValidPath(uri.getRawPath()) || !isValidQuery(uri.getRawQuery()) || !isValidFragment(uri.getRawFragment())) {
    395            return false;
    396        }
    397        return true;
    398    }
    399
    400    /**
    401     * Returns true if the authority is properly formatted.  An authority is the combination
    402     * of hostname and port.  A {@code null} authority value is considered invalid.
    403     * Note: this implementation validates the domain unless a RegexValidator was provided.
    404     * If a RegexValidator was supplied, and it matches, then the authority is regarded
    405     * as valid with no further checks, otherwise the method checks against the
    406     * AUTHORITY_PATTERN and the DomainValidator (ALLOW_LOCAL_URLS)
    407     *
    408     * @param authority Authority value to validate, allows IDN
    409     * @return true if authority (hostname and port) is valid.
    410     */
    411    protected boolean isValidAuthority(final String authority) {
    412        if (authority == null) {
    413            return false;
    414        }
    415
    416        // check manual authority validation if specified
    417        if (authorityValidator != null && authorityValidator.isValid(authority)) {
    418            return true;
    419        }
    420        // convert to ASCII if possible
    421        final String authorityASCII = DomainValidator.unicodeToASCII(authority);
    422
    423        final Matcher authorityMatcher = AUTHORITY_PATTERN.matcher(authorityASCII);
    424        if (!authorityMatcher.matches()) {
    425            return false;
    426        }
    427
    428        // We have to process IPV6 separately because that is parsed in a different group
    429        final String ipv6 = authorityMatcher.group(PARSE_AUTHORITY_IPV6);
    430        if (ipv6 != null) {
    431            final InetAddressValidator inetAddressValidator = InetAddressValidator.getInstance();
    432            if (!inetAddressValidator.isValidInet6Address(ipv6)) {
    433                return false;
    434            }
    435        } else {
    436            final String hostLocation = authorityMatcher.group(PARSE_AUTHORITY_HOST_IP);
    437            // check if authority is hostname or IP address:
    438            // try a hostname first since that's much more likely
    439            if (!domainValidator.isValid(hostLocation)) {
    440                // try an IPv4 address
    441                final InetAddressValidator inetAddressValidator = InetAddressValidator.getInstance();
    442                if (!inetAddressValidator.isValidInet4Address(hostLocation)) {
    443                    // isn't IPv4, so the URL is invalid
    444                    return false;
    445                }
    446            }
    447        }
    448
    449        // the port is captured in the same group regardless of host form, so it must be
    450        // range checked for a bracketed IPv6 host too, not just the hostname/IPv4 branch
    451        final String port = authorityMatcher.group(PARSE_AUTHORITY_PORT);
    452        if (!GenericValidator.isBlankOrNull(port)) {
    453            try {
    454                final int iPort = Integer.parseInt(port);
    455                if (iPort < 0 || iPort > MAX_UNSIGNED_16_BIT_INT) {
    456                    return false;
    457                }
    458            } catch (final NumberFormatException nfe) {
    459                return false; // this can happen for big numbers
    460            }
    461        }
    462
    463        final String extra = authorityMatcher.group(PARSE_AUTHORITY_EXTRA);
    464        if (extra != null && !extra.trim().isEmpty()) {
    465            return false;
    466        }
    467
    468        return true;
    469    }
    470
    471    /**
    472     * Returns true if the given fragment is null or fragments are allowed.
    473     *
    474     * @param fragment Fragment value to validate.
    475     * @return true if fragment is valid.
    476     */
    477    protected boolean isValidFragment(final String fragment) {
    478        if (fragment == null) {
    479            return true;
    480        }
    481
    482        return isOff(NO_FRAGMENTS);
    483    }
    484
    485    /**
    486     * Returns true if the path is valid.  A {@code null} value is considered invalid.
    487     *
    488     * @param path Path value to validate.
    489     * @return true if path is valid.
    490     */
    491    protected boolean isValidPath(final String path) {
    492        if (path == null || !PATH_PATTERN.matcher(path).matches()) {
    493            return false;
    494        }
    495
    496        try {
    497            // This will convert '+' to space, but that is irrelevant for the purpose of validation
    498            final String decodedPath = URLDecoder.decode(path, StandardCharsets.UTF_8.name());
    499            // Don't omit host otherwise leading path may be taken as host if it starts with //
    500            final URI uri = new URI(null, "localhost", decodedPath, null);
    501            final String norm = uri.normalize().getPath();
    502            if (norm.startsWith("/../") // Trying to go via the parent dir
    503                    || norm.equals("/..")) { // Trying to go to the parent dir
    504                return false;
    505            }
    506            final int slash2Count = countToken("//", decodedPath);
    507            if (isOff(ALLOW_2_SLASHES) && slash2Count > 0) {
    508                return false;
    509            }
    510        } catch (final UnsupportedEncodingException | IllegalArgumentException | URISyntaxException e) {
    511            return false;
    512        }
    513
    514        return true;
    515    }
    516
    517    /**
    518     * Returns true if the query is null, or it's a properly formatted query string.
    519     *
    520     * @param query Query value to validate.
    521     * @return true if query is valid.
    522     */
    523    protected boolean isValidQuery(final String query) {
    524        if (query == null) {
    525            return true;
    526        }
    527        return QUERY_PATTERN.matcher(query).matches();
    528    }
    529
    530    /**
    531     * Validate scheme. If schemes[] was initialized to a non-null,
    532     * then only those schemes are allowed.
    533     * Otherwise, the default schemes are "http", "https", "ftp".
    534     * Matching is case-blind.
    535     *
    536     * @param scheme The scheme to validate.  A {@code null} value is considered
    537     * invalid.
    538     * @return true if valid.
    539     */
    540    protected boolean isValidScheme(final String scheme) {
    541        if (scheme == null || !SCHEME_PATTERN.matcher(scheme).matches()
    542                || isOff(ALLOW_ALL_SCHEMES) && !allowedSchemes.contains(scheme.toLowerCase(Locale.ENGLISH))) {
    543            return false;
    544        }
    545
    546        return true;
    547    }
    548
    549}