Add fix for Windows caching of pip (#332)

.github/workflows/e2e-cache.yml

@@ -21,7 +21,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        python-version: ['3.7', '3.8', '3.9', 'pypy-3.7-v7.3.5', 'pypy-3.7-v7.x']
+        python-version: ['3.8', '3.9', 'pypy-3.7-v7.x']
     steps:
       - uses: actions/checkout@v2
       - name: Setup Python
@@ -39,18 +39,18 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        python-version: ['3.7', '3.8', '3.9', 'pypy-3.7-v7.3.5', 'pypy-3.7-v7.x']
+        python-version: ['3.8', '3.9', 'pypy-3.7-v7.x']
     steps:
       - uses: actions/checkout@v2
-      - name: Install pipenv
-        run: pipx install pipenv
       - name: Setup Python
         uses: ./
         with:
           python-version: ${{ matrix.python-version }}
           cache: 'pipenv'
+      - name: Install pipenv
+        run: pipx install pipenv
       - name: Install dependencies
-        run: pipenv install flake8
+        run: pipenv install numpy
 
   python-pip-dependencies-caching-path:
     name: Test pip (Python ${{ matrix.python-version}}, ${{ matrix.os }})
@@ -59,7 +59,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        python-version: ['3.7', '3.8', '3.9', 'pypy-3.7-v7.3.5', 'pypy-3.7-v7.x']
+        python-version: ['3.8', '3.9', 'pypy-3.7-v7.x']
     steps:
       - uses: actions/checkout@v2
       - name: Setup Python
@@ -78,16 +78,16 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        python-version: ['3.7', '3.8', '3.9', 'pypy-3.7-v7.3.5', 'pypy-3.7-v7.x']
+        python-version: ['3.8', '3.9', 'pypy-3.7-v7.x']
     steps:
       - uses: actions/checkout@v2
-      - name: Install pipenv
-        run: pipx install pipenv
       - name: Setup Python
         uses: ./
         with:
           python-version: ${{ matrix.python-version }}
           cache: 'pipenv'
           cache-dependency-path: '**/requirements-linux.txt'
+      - name: Install pipenv
+        run: pipx install pipenv
       - name: Install dependencies
-        run: pipenv install flake8
+        run: pipenv install numpy

.github/workflows/licensed.yml

@@ -14,11 +14,15 @@ jobs:
     name: Check licenses
     steps:
       - uses: actions/checkout@v2
+      - name: Set Node.js 12.x
+        uses: actions/setup-node@v2
+        with:
+          node-version: 12.x
       - run: npm ci
       - name: Install licensed
         run: |
           cd $RUNNER_TEMP
-          curl -Lfs -o licensed.tar.gz https://github.com/github/licensed/releases/download/2.12.2/licensed-2.12.2-linux-x64.tar.gz
+          curl -Lfs -o licensed.tar.gz https://github.com/github/licensed/releases/download/3.3.1/licensed-3.3.1-linux-x64.tar.gz
           sudo tar -xzf licensed.tar.gz
           sudo mv licensed /usr/local/bin/licensed
       - run: licensed status

.github/workflows/test-python.yml

@@ -97,7 +97,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [macos-10.15, windows-latest, ubuntu-18.04, ubuntu-20.04]
+        os: [macos-10.15, windows-2019, ubuntu-18.04, ubuntu-20.04]
     steps:
     - name: Checkout
       uses: actions/checkout@v2

.licenses/npm/node-fetch.dep.yml

@@ -1,6 +1,6 @@
 ---
 name: node-fetch
-version: 2.6.6
+version: 2.6.7
 type: npm
 summary: A light-weight module that brings window.fetch to node.js
 homepage: https://github.com/bitinn/node-fetch

README.md

@@ -43,7 +43,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ '2.x', '3.x', 'pypy-2.7', 'pypy-3.6', 'pypy-3.7' ]
+        python-version: [ '2.x', '3.x', 'pypy-2.7', 'pypy-3.7', 'pypy-3.8' ]
     name: Python ${{ matrix.python-version }} sample
     steps:
       - uses: actions/checkout@v2
@@ -63,7 +63,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: ['2.7', '3.6', '3.7', '3.8', 'pypy-2.7', 'pypy-3.6']
+        python-version: ['2.7', '3.7', '3.8', '3.9', '3.10', 'pypy-2.7', 'pypy-3.8']
         exclude:
           - os: macos-latest
             python-version: '3.8'
@@ -76,7 +76,7 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
       - name: Display Python version
-        run: python -c "import sys; print(sys.version)"
+        run: python --version
 ```
 
 Download and set up a version of Python that does not come preinstalled on an image:
@@ -87,7 +87,7 @@ jobs:
     strategy:
       matrix:
         # in this example, there is a newer version already installed, 3.7.7, so the older version will be downloaded
-        python-version: ['3.5', '3.6', '3.7.4', '3.8']
+        python-version: ['3.7.4', '3.8', '3.9', '3.10']
     steps:
     - uses: actions/checkout@v2
     - uses: actions/setup-python@v2
@@ -102,7 +102,7 @@ steps:
 - uses: actions/checkout@v2
 - uses: actions/setup-python@v2
   with:
-    python-version: '3.9.0-beta.4'
+    python-version: '3.11.0-alpha.1'
 - run: python my_script.py
 ```
 
@@ -112,7 +112,7 @@ steps:
 - uses: actions/checkout@v2
 - uses: actions/setup-python@v2
   with:
-    python-version: '3.9.0-alpha - 3.9.0' # SemVer's version range syntax
+    python-version: '3.11.0-alpha - 3.11.0' # SemVer's version range syntax
 - run: python my_script.py
 ```
 
@@ -125,9 +125,9 @@ jobs:
     strategy:
       matrix:
         python-version:
-        - 'pypy-3.6' # the latest available version of PyPy that supports Python 3.6
         - 'pypy-3.7' # the latest available version of PyPy that supports Python 3.7
         - 'pypy-3.7-v7.3.3' # Python 3.7 and PyPy 7.3.3
+        - 'pypy-3.8' # the latest available version of PyPy that supports Python 3.8
     steps:
     - uses: actions/checkout@v2
     - uses: actions/setup-python@v2
@@ -146,7 +146,7 @@ Check out our detailed guide on using [Python with GitHub Actions](https://help.
 `setup-python` is able to configure Python from two sources:
 
 - Preinstalled versions of Python in the tools cache on GitHub-hosted runners.
-    - For detailed information regarding the available versions of Python that are installed see [Supported software](https://docs.github.com/en/actions/reference/specifications-for-github-hosted-runners#supported-software).
+    - For detailed information regarding the available versions of Python that are installed, see [Supported software](https://docs.github.com/en/actions/reference/specifications-for-github-hosted-runners#supported-software).
     - For every minor version of Python, expect only the latest patch to be preinstalled.
     - If `3.8.1` is installed for example, and `3.8.2` is released, expect `3.8.1` to be removed and replaced by `3.8.2` in the tools cache.
     - If the exact patch version doesn't matter to you, specifying just the major and minor version will get you the latest preinstalled patch version. In the previous example, the version spec `3.8` will use the `3.8.2` Python version found in the cache.
@@ -159,9 +159,9 @@ Check out our detailed guide on using [Python with GitHub Actions](https://help.
  `setup-python` is able to configure PyPy from two sources:
 
 - Preinstalled versions of PyPy in the tools cache on GitHub-hosted runners
-  - For detailed information regarding the available versions of PyPy that are installed see [Supported software](https://docs.github.com/en/actions/reference/specifications-for-github-hosted-runners#supported-software).
+  - For detailed information regarding the available versions of PyPy that are installed, see [Supported software](https://docs.github.com/en/actions/reference/specifications-for-github-hosted-runners#supported-software).
   - For the latest PyPy release, all versions of Python are cached.
-  - Cache is updated with a 1-2 week delay. If you specify the PyPy version as `pypy-3.6`, the cached version will be used although a newer version is available. If you need to start using the recently released version right after release, you should specify the exact PyPy version using `pypy-3.6-v7.3.3`.
+  - Cache is updated with a 1-2 week delay. If you specify the PyPy version as `pypy-3.7`, the cached version will be used although a newer version is available. If you need to start using the recently released version right after release, you should specify the exact PyPy version using `pypy-3.7-v7.3.3`.
 
 - Downloadable PyPy versions from the [official PyPy site](https://downloads.python.org/pypy/).
   - All available versions that we can download are listed in [versions.json](https://downloads.python.org/pypy/versions.json) file.
@@ -198,8 +198,8 @@ The version of PyPy should be specified in the format `pypy-<python_version>[-v<
 The `<pypy_version>` parameter is optional and can be skipped. The latest version will be used in this case.
 
 ```
-pypy-3.6 # the latest available version of PyPy that supports Python 3.6
 pypy-3.7 # the latest available version of PyPy that supports Python 3.7
+pypy-3.8 # the latest available version of PyPy that supports Python 3.8
 pypy-2.7 # the latest available version of PyPy that supports Python 2.7
 pypy-3.7-v7.3.3 # Python 3.7 and PyPy 7.3.3
 pypy-3.7-v7.x # Python 3.7 and the latest available PyPy 7.x
@@ -230,7 +230,6 @@ steps:
     python-version: '3.9'
     cache: 'pip'
 - run: pip install -r requirements.txt
-- run: pip test
 ```
 
 **Caching pipenv dependencies:**
@@ -244,7 +243,6 @@ steps:
     python-version: '3.9'
     cache: 'pipenv'
 - run: pipenv install
-- run: pipenv test
 ```
 
 **Using wildcard patterns to cache dependencies**
@@ -257,7 +255,6 @@ steps:
     cache: 'pip'
     cache-dependency-path: '**/requirements-dev.txt'
 - run: pip install -r subdirectory/requirements-dev.txt
-- run: pip test
 ```
 
 **Using a list of file paths to cache dependencies**
@@ -274,7 +271,6 @@ steps:
       server/app/Pipfile.lock
       __test__/app/Pipfile.lock
 - run: pipenv install
-- run: pipenv test
 ```
 
 # Using `setup-python` with a self hosted runner
@@ -304,7 +300,7 @@ If you are experiencing problems while configuring Python on your self-hosted ru
   - The user starting the runner is in the owning group, and the owning group has write permission.
   - All users have write permission.
 - One quick way to grant access is to change the user and group of `/opt/hostedtoolcache` to be the same as the runners using `chown`.
-    - `sudo chown runner-user:runner-group opt/hostedtoolcache/`.
+    - `sudo chown runner-user:runner-group /opt/hostedtoolcache/`.
 - If your runner is configured as a service and you run into problems, make sure the user that the service is running as is correct. For more information, you can [check the status of your self-hosted runner](https://help.github.com/en/actions/hosting-your-own-runners/configuring-the-self-hosted-runner-application-as-a-service#checking-the-status-of-the-service).
 
 ### Mac

__tests__/cache-restore.test.ts

@@ -92,15 +92,12 @@ describe('restore-cache', () => {
           dependencyFile
         );
         await cacheDistributor.restoreCache();
-        let pythonKey = '';
-        if (packageManager === 'pipenv') {
-          pythonKey = `python-${pythonVersion}-`;
-        }
 
         expect(infoSpy).toHaveBeenCalledWith(
-          `Cache restored from key: setup-python-${process.env['RUNNER_OS']}-${pythonKey}${packageManager}-${fileHash}`
+          `Cache restored from key: setup-python-${process.env['RUNNER_OS']}-python-${pythonVersion}-${packageManager}-${fileHash}`
         );
-      }
+      },
+      30000
     );
 
     it.each([

dist/cache-save/index.js

@@ -34585,9 +34585,17 @@ AbortError.prototype = Object.create(Error.prototype);
 AbortError.prototype.constructor = AbortError;
 AbortError.prototype.name = 'AbortError';
 
+const URL$1 = Url.URL || whatwgUrl.URL;
+
 // fix an issue where "PassThrough", "resolve" aren't a named export for node <10
 const PassThrough$1 = Stream.PassThrough;
-const resolve_url = Url.resolve;
+
+const isDomainOrSubdomain = function isDomainOrSubdomain(destination, original) {
+	const orig = new URL$1(original).hostname;
+	const dest = new URL$1(destination).hostname;
+
+	return orig === dest || orig[orig.length - dest.length - 1] === '.' && orig.endsWith(dest);
+};
 
 /**
  * Fetch function
@@ -34675,7 +34683,19 @@ function fetch(url, opts) {
 				const location = headers.get('Location');
 
 				// HTTP fetch step 5.3
-				const locationURL = location === null ? null : resolve_url(request.url, location);
+				let locationURL = null;
+				try {
+					locationURL = location === null ? null : new URL$1(location, request.url).toString();
+				} catch (err) {
+					// error here can only be invalid URL in Location: header
+					// do not throw when options.redirect == manual
+					// let the user extract the errorneous redirect URL
+					if (request.redirect !== 'manual') {
+						reject(new FetchError(`uri requested responds with an invalid redirect URL: ${location}`, 'invalid-redirect'));
+						finalize();
+						return;
+					}
+				}
 
 				// HTTP fetch step 5.5
 				switch (request.redirect) {
@@ -34723,6 +34743,12 @@ function fetch(url, opts) {
 							size: request.size
 						};
 
+						if (!isDomainOrSubdomain(request.url, locationURL)) {
+							for (const name of ['authorization', 'www-authenticate', 'cookie', 'cookie2']) {
+								requestOpts.headers.delete(name);
+							}
+						}
+
 						// HTTP-redirect fetch step 9
 						if (res.statusCode !== 303 && request.body && getTotalBytes(request) === null) {
 							reject(new FetchError('Cannot follow redirect with body being a readable stream', 'unsupported-redirect'));

dist/setup/index.js

@@ -34463,16 +34463,38 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const glob = __importStar(__webpack_require__(281));
 const core = __importStar(__webpack_require__(470));
 const exec = __importStar(__webpack_require__(986));
+const child_process = __importStar(__webpack_require__(129));
+const util_1 = __importDefault(__webpack_require__(669));
 const path = __importStar(__webpack_require__(622));
 const os_1 = __importDefault(__webpack_require__(87));
 const cache_distributor_1 = __importDefault(__webpack_require__(435));
+const utils_1 = __webpack_require__(163);
 class PipCache extends cache_distributor_1.default {
-    constructor(cacheDependencyPath = '**/requirements.txt') {
+    constructor(pythonVersion, cacheDependencyPath = '**/requirements.txt') {
         super('pip', cacheDependencyPath);
+        this.pythonVersion = pythonVersion;
     }
     getCacheGlobalDirectories() {
         return __awaiter(this, void 0, void 0, function* () {
-            const { stdout, stderr, exitCode } = yield exec.getExecOutput('pip cache dir');
+            let exitCode = 1;
+            let stdout = '';
+            let stderr = '';
+            // Add temporary fix for Windows
+            // On windows it is necessary to execute through an exec
+            // because the getExecOutput gives a non zero code or writes to stderr for pip 22.0.2,
+            // or spawn must be started with the shell option enabled for getExecOutput
+            // Related issue: https://github.com/actions/setup-python/issues/328
+            if (utils_1.IS_WINDOWS) {
+                const execPromisify = util_1.default.promisify(child_process.exec);
+                ({ stdout: stdout, stderr: stderr } = yield execPromisify('pip cache dir'));
+            }
+            else {
+                ({
+                    stdout: stdout,
+                    stderr: stderr,
+                    exitCode: exitCode
+                } = yield exec.getExecOutput('pip cache dir'));
+            }
             if (exitCode && stderr) {
                 throw new Error(`Could not get cache folder path for pip package manager`);
             }
@@ -34487,8 +34509,8 @@ class PipCache extends cache_distributor_1.default {
     computeKeys() {
         return __awaiter(this, void 0, void 0, function* () {
             const hash = yield glob.hashFiles(this.cacheDependencyPath);
-            const primaryKey = `${this.CACHE_KEY_PREFIX}-${process.env['RUNNER_OS']}-${this.packageManager}-${hash}`;
-            const restoreKey = `${this.CACHE_KEY_PREFIX}-${process.env['RUNNER_OS']}-${this.packageManager}`;
+            const primaryKey = `${this.CACHE_KEY_PREFIX}-${process.env['RUNNER_OS']}-python-${this.pythonVersion}-${this.packageManager}-${hash}`;
+            const restoreKey = `${this.CACHE_KEY_PREFIX}-${process.env['RUNNER_OS']}-python-${this.pythonVersion}-${this.packageManager}`;
             return {
                 primaryKey,
                 restoreKey: [restoreKey]
@@ -37136,9 +37158,17 @@ AbortError.prototype = Object.create(Error.prototype);
 AbortError.prototype.constructor = AbortError;
 AbortError.prototype.name = 'AbortError';
 
+const URL$1 = Url.URL || whatwgUrl.URL;
+
 // fix an issue where "PassThrough", "resolve" aren't a named export for node <10
 const PassThrough$1 = Stream.PassThrough;
-const resolve_url = Url.resolve;
+
+const isDomainOrSubdomain = function isDomainOrSubdomain(destination, original) {
+	const orig = new URL$1(original).hostname;
+	const dest = new URL$1(destination).hostname;
+
+	return orig === dest || orig[orig.length - dest.length - 1] === '.' && orig.endsWith(dest);
+};
 
 /**
  * Fetch function
@@ -37226,7 +37256,19 @@ function fetch(url, opts) {
 				const location = headers.get('Location');
 
 				// HTTP fetch step 5.3
-				const locationURL = location === null ? null : resolve_url(request.url, location);
+				let locationURL = null;
+				try {
+					locationURL = location === null ? null : new URL$1(location, request.url).toString();
+				} catch (err) {
+					// error here can only be invalid URL in Location: header
+					// do not throw when options.redirect == manual
+					// let the user extract the errorneous redirect URL
+					if (request.redirect !== 'manual') {
+						reject(new FetchError(`uri requested responds with an invalid redirect URL: ${location}`, 'invalid-redirect'));
+						finalize();
+						return;
+					}
+				}
 
 				// HTTP fetch step 5.5
 				switch (request.redirect) {
@@ -37274,6 +37316,12 @@ function fetch(url, opts) {
 							size: request.size
 						};
 
+						if (!isDomainOrSubdomain(request.url, locationURL)) {
+							for (const name of ['authorization', 'www-authenticate', 'cookie', 'cookie2']) {
+								requestOpts.headers.delete(name);
+							}
+						}
+
 						// HTTP-redirect fetch step 9
 						if (res.statusCode !== 303 && request.body && getTotalBytes(request) === null) {
 							reject(new FetchError('Cannot follow redirect with body being a readable stream', 'unsupported-redirect'));
@@ -43888,7 +43936,7 @@ var PackageManagers;
 function getCacheDistributor(packageManager, pythonVersion, cacheDependencyPath) {
     switch (packageManager) {
         case PackageManagers.Pip:
-            return new pip_cache_1.default(cacheDependencyPath);
+            return new pip_cache_1.default(pythonVersion, cacheDependencyPath);
         case PackageManagers.Pipenv:
             return new pipenv_cache_1.default(pythonVersion, cacheDependencyPath);
         default:

package-lock.json

@@ -8137,9 +8137,9 @@
       "dev": true
     },
     "node-fetch": {
-      "version": "2.6.6",
-      "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.6.tgz",
-      "integrity": "sha512-Z8/6vRlTUChSdIgMa51jxQ4lrw/Jy5SOW10ObaA47/RElsAN2c5Pn8bTgFGWn/ibwzXTE8qwr1Yzx28vsecXEA==",
+      "version": "2.6.7",
+      "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz",
+      "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==",
       "requires": {
         "whatwg-url": "^5.0.0"
       },

package.json

@@ -5,7 +5,7 @@
   "description": "Setup python action",
   "main": "dist/index.js",
   "scripts": {
-    "build": "tsc",
+    "build": "ncc build -o dist/setup src/setup-python.ts && ncc build -o dist/cache-save src/cache-save.ts",
     "format": "prettier --write \"{,!(node_modules)/**/}*.ts\"",
     "format-check": "prettier --check \"{,!(node_modules)/**/}*.ts\"",
     "release": "ncc build -o dist/setup src/setup-python.ts && ncc build -o dist/cache-save src/cache-save.ts && git add -f dist/",

src/cache-distributions/cache-factory.ts

@@ -13,7 +13,7 @@ export function getCacheDistributor(
 ) {
   switch (packageManager) {
     case PackageManagers.Pip:
-      return new PipCache(cacheDependencyPath);
+      return new PipCache(pythonVersion, cacheDependencyPath);
     case PackageManagers.Pipenv:
       return new PipenvCache(pythonVersion, cacheDependencyPath);
     default:

src/cache-distributions/pip-cache.ts

@@ -1,21 +1,42 @@
 import * as glob from '@actions/glob';
 import * as core from '@actions/core';
 import * as exec from '@actions/exec';
-
+import * as child_process from 'child_process';
+import utils from 'util';
 import * as path from 'path';
 import os from 'os';
 
 import CacheDistributor from './cache-distributor';
+import {IS_WINDOWS} from '../utils';
 
 class PipCache extends CacheDistributor {
-  constructor(cacheDependencyPath: string = '**/requirements.txt') {
+  constructor(
+    private pythonVersion: string,
+    cacheDependencyPath: string = '**/requirements.txt'
+  ) {
     super('pip', cacheDependencyPath);
   }
 
   protected async getCacheGlobalDirectories() {
-    const {stdout, stderr, exitCode} = await exec.getExecOutput(
-      'pip cache dir'
-    );
+    let exitCode = 1;
+    let stdout = '';
+    let stderr = '';
+
+    // Add temporary fix for Windows
+    // On windows it is necessary to execute through an exec
+    // because the getExecOutput gives a non zero code or writes to stderr for pip 22.0.2,
+    // or spawn must be started with the shell option enabled for getExecOutput
+    // Related issue: https://github.com/actions/setup-python/issues/328
+    if (IS_WINDOWS) {
+      const execPromisify = utils.promisify(child_process.exec);
+      ({stdout: stdout, stderr: stderr} = await execPromisify('pip cache dir'));
+    } else {
+      ({
+        stdout: stdout,
+        stderr: stderr,
+        exitCode: exitCode
+      } = await exec.getExecOutput('pip cache dir'));
+    }
 
     if (exitCode && stderr) {
       throw new Error(
@@ -36,8 +57,8 @@ class PipCache extends CacheDistributor {
 
   protected async computeKeys() {
     const hash = await glob.hashFiles(this.cacheDependencyPath);
-    const primaryKey = `${this.CACHE_KEY_PREFIX}-${process.env['RUNNER_OS']}-${this.packageManager}-${hash}`;
-    const restoreKey = `${this.CACHE_KEY_PREFIX}-${process.env['RUNNER_OS']}-${this.packageManager}`;
+    const primaryKey = `${this.CACHE_KEY_PREFIX}-${process.env['RUNNER_OS']}-python-${this.pythonVersion}-${this.packageManager}-${hash}`;
+    const restoreKey = `${this.CACHE_KEY_PREFIX}-${process.env['RUNNER_OS']}-python-${this.pythonVersion}-${this.packageManager}`;
 
     return {
       primaryKey,