Migrate TeamCity builtin artifact storage to Google Storage Buckets (Google Artifact Storage Plugin)

convert_tc_folder.py

# -*- coding: utf-8 -*-
"""
Convert old teamcity artifacts folder structure to Google Artifact Storage structure.
Use at your own risk and please CREATE A BACKUP before using this!

Written by:
    Frank van den Hoek<[email protected]>

Copyright 2019 Trancon B.V.

TeamCity by default stores artifacts in a folder structure like below

- <Project name>
    - <Build config name>
        - <Build id>
            - .teamcity
                - logs
                  properties
                  settings
              <artifacts..>


After you install the Google Artifact Storage plugin from the link below 
the structure changes slightly. https://plugins.jetbrains.com/plugin/9634-google-artifact-storage 
The artifacts are gone (moved to the Google Bucket) but the .teamcity folder remains but now contains
a artifacts.json file with the contents of the folder at the Google Storage Bucket for fast lookup.

The folder structure at the Storage Bucket level looks like this:

- <Project name>
    - <Build configuration id>
        - <Build id>
            <artifacts..>

This script moves the .teamcity folder in the current (default TeamCity) storage folder (src_dir) 
to a new folder (target_dir) and places an artifacts.json file in the .teamcity folder in the target_dir. 
When finished the src_dir can be uploaded to the Google Storage Bucket. 
The target_dir is the new system/artifacts dir for TeamCity. 

Example artifacts.json file:
{
  "version": "2017.1",
  "storage_settings_id": "PROJECT_EXT_5",
  "properties": {
    "google_path_prefix": "Project/build config id/build id"
  },
  "artifacts": [
    {
      "path": "sub/folder/folder/file-demo.xml",
      "size": 4478,
      "properties": {}
    },..
  ]
}

Usage:
python convert_tc_folder.py <version> <storage_settings_id> <src_dir> <target_dir>

examples:
python convert_tc_folder.py 2017.1 PROJECT_EXT_5 ./artifacts ~/artifacts_for_google/

"""
import sys
import os
import json
import shutil


# EDIT before you run it
build_config_name_to_build_config_id = {
    "project/build_config_name": "build_config_id"
}


def get_target_build_config_dir(project, build_config_name):
    """

    :param project:
    :param build_config_name:
    :return:
    """
    build_key = "{}/{}".format(project, build_config_name)
    build = build_config_name
    if build_key in build_config_name_to_build_config_id:
        build = build_config_name_to_build_config_id[build_key]
    return build


def migrate_build_dir(build_dir_path, target_path):
    ''' Migrates the .teamcity folder to target_path

    :param build_dir_path:string
    :return:
    '''
    shutil.move(os.path.join(build_dir_path, '.teamcity'), os.path.join(target_path, '.teamcity'))


def get_recursive_file_list(build_dir_path):
    """ Get's the complete filelist of entire folder

    :param build_dir_path:string
    :return:
    """
    files = []
    for root, dirnames, filenames in os.walk(build_dir_path):
        if "{}.teamcity".format(os.sep) in root:
            continue
        for filename in filenames:
            files.append(os.path.join(root, filename))
    return files


def create_artifact_json(build_dir_path, version, storage_settings_id):
    """ Creates the artifact.json in the .teamcity dir

    :param build_dir_path:string
    :return:
    """

    path_parts = build_dir_path.split(os.sep)
    #to_replace = os.sep.join(path_parts[0:-3]) + os.sep
    project, build, build_id = tuple(path_parts[-3:])
    file_paths = get_recursive_file_list(build_dir_path)
    artifacts = []

    correct_build = get_target_build_config_dir(project, build)

    for file_path in file_paths:
        file_stats = os.stat(file_path)
        artifacts.append({
            "path": file_path.replace(build_dir_path + os.sep, '').replace("\\", "/"),
            "size": file_stats.st_size,
            "properties": {}
        })


    json_dict = {
        "version": version,
        "storage_settings_id": storage_settings_id,
        "properties": {
            "google_path_prefix": "{0}/{1}/{2}".format(project, correct_build, build_id)
        },
        "artifacts": artifacts
    }

    with open(os.path.join(build_dir_path, ".teamcity", "artifacts.json"), "w") as artifact_file:
        artifact_file.write(json.dumps(json_dict, indent=4))


def main(version, storage_settings_id, src_dir, target_dir):
    """

    :param version: string - version to put it json file
    :param storage_settings_id: string - the id of the storage setting on the project
    :param src_dir: string - path to source directory (current teamcity artifact dir)
    :param target_dir: - path to target directory (new teamcity artifact dir)
    :return:
    """
    folders_to_rename = {}
    if not storage_settings_id \
            or not os.path.isdir(src_dir) \
            or not os.path.isdir(target_dir):
        raise ValueError("Arguments are invalid")

    if not version: version = '2017.1'

    safety_switch = ""
    for root, dirs, _ in os.walk(src_dir):
        if root.count(os.sep) == 3:
            for dir in dirs:
                # determine src and target folder
                src = os.path.join(root, dir)
                project, build = tuple(root.split(os.sep)[-2:])
                correct_build = get_target_build_config_dir(project, build)
                target = os.path.join(target_dir, os.sep.join([project, build]), dir)
                # elect folder for rename
                folders_to_rename[root] = os.sep.join(root.split(os.sep)[:-1] + [correct_build])

                if not os.path.isdir(os.path.join(src, '.teamcity')):
                    print("Skipping (no .teamcity folder) {}".format(src))
                    continue

                print("Converting {} to {}".format(src, target))

                # create the artifact.json file
                create_artifact_json(src, version, storage_settings_id)
                # migrates the .teamcity folder to target
                migrate_build_dir(src, target)

                if safety_switch != "ok":
                    safety_switch = raw_input("Type (ok/exit):")
                    if safety_switch == "exit":
                        return

    if raw_input("Rename src folder? (yes/no):") == "yes":
        for src, target in folders_to_rename.iteritems():
            os.rename(src, target)


if __name__ == '__main__':
    usage = "convert_tc_folder.py [Convert teamcity folder]\n" \
            "\n" \
            "Basic usage:\n" \
            "Test:   convert_tc_folder.py <version> <storage_settings_id> <src_dir> <target_dir>\n" \
            "Help:   convert_tc_folder.py -h\n"
    try:
        if len(sys.argv) != 5 or sys.argv[1] in ['-h', '-H']:
            print(usage)
        else:
            main(*tuple(sys.argv[1:]))
    except IndexError as error:
        print(usage)

Look up storage settings id in TeamcCity ui. Go to project>Artifacts Storage and click edit. The id is now in the url.

Descend into folder above artifacts folder (normally $TEAMCITY_HOME/system). Create a new folder called artifacts_new or whatever you prefer. Run convert command below

./convert_tc_folder.py 2017.1 PROJECT_EXT_5 ./artifacts ./artifacts_new

Good luck!