diff --git a/P4Transfer.py b/P4Transfer.py index 1fb042c..5a18d9f 100755 --- a/P4Transfer.py +++ b/P4Transfer.py @@ -211,7 +211,8 @@ class P4TConfigException(P4TException): superuser: "y" source: - # P4PORT to connect to, e.g. some-server:1666 + # P4PORT to connect to, e.g. some-server:1666 - if this is on localhost and you just + # want to specify port number, then use quotes: "1666" p4port: # P4USER to use p4user: @@ -219,12 +220,14 @@ class P4TConfigException(P4TException): p4client: # P4PASSWD for the user - valid password. If blank then no login performed. # Recommended to make sure user is in a group with a long password timeout!. + # Make sure your P4TICKETS file is correctly found in the environment p4passwd: - # P4CHARSET to use, e.g. none, utf8, etc + # P4CHARSET to use, e.g. none, utf8, etc - leave blank for non-unicode p4d instance p4charset: target: - # P4PORT to connect to, e.g. some-server:1666 + # P4PORT to connect to, e.g. some-server:1666 - if this is on localhost and you just + # want to specify port number, then use quotes: "1666" p4port: # P4USER to use p4user: @@ -232,13 +235,15 @@ class P4TConfigException(P4TException): p4client: # P4PASSWD for the user - valid password. If blank then no login performed. # Recommended to make sure user is in a group with a long password timeout! + # Make sure your P4TICKETS file is correctly found in the environment p4passwd: - # P4CHARSET to use, e.g. none, utf8, etc + # P4CHARSET to use, e.g. none, utf8, etc - leave blank for non-unicode p4d instance p4charset: # workspace_root: Root directory to use for both client workspaces. # This will be used to update the client workspace Root: field for both source/target workspaces # They must be the same. +# Make sure there is enough space to hold the largest single changelist that will be transferred! workspace_root: /work/transfer # views: An array of source/target view mappings diff --git a/doc/P4Transfer.adoc b/doc/P4Transfer.adoc index 4ea2f89..d366047 100644 --- a/doc/P4Transfer.adoc +++ b/doc/P4Transfer.adoc @@ -1,7 +1,7 @@ = P4Transfer - Helix Core Full History Migration Tool Perforce Professional Services :revnumber: v2021.1 -:revdate: 2021-02-22 +:revdate: 2021-04-02 :doctype: book :icons: font :toc: @@ -128,6 +128,8 @@ The easiest thing to do is to download this repo either by: The minimum requirements are the modules `P4Transfer.py` and `logutils.py` +If you have installed P4Python as above, then check the `requirements.txt` for other modules to install via `pip` or `pip3`. + === Getting started Note that if running it on Windows, and especially if the source server has filenames containing say umlauts or other non-ASCII characters, then Python 2.7 is required currently due to the way Unicode is processed. Python 3.6+ on Mac/Unix should be fine with Unicode as long as you are using P4Python 2017.2+ @@ -136,9 +138,9 @@ Create the workspaces for both servers, ensuring that the root directories and c Now initialize the configuration file, by default called `transfer.cfg`. This can be generated by the script: - python3 P4Transfer.py –-sample-config > transfer.cg + python3 P4Transfer.py –-sample-config > transfer.yaml -Then edit the resulting file. +Then edit the resulting file, paying attention to the comments. The password stored in P4Passwd is optional if you do not want to rely on tickets. The tool performs a login if provided with a password, so it should work with `security=3` or `auth_check` trigger set. @@ -158,6 +160,9 @@ If there are any changes missing, they will be applied consecutively. P4Transfer has various options – these are documented via the `-h` or `--help` parameters. +The following text may not display properly if your are viewing this P4Transfer.adoc file in GitHub. Please refer +the the .pdf version instead, or open up link:help.txt[help.txt] file directly. + [source] ---- include::help.txt[] @@ -178,12 +183,36 @@ On Linux, recommend you execute it as a background process, and then monitor the This will run in the background, and poll for new changes (according to `poll_interval` in the config file.) -You can look at the output file for progress, e.g. (avoiding long lines of text which can be output) +You can look at the output file for progress, e.g. (avoiding long lines of text which can be output), or grep the log file: tail -f out1 | cut -c -140 + grep :INFO: log-P4Transfer-*.log + Note that if you edit the configuration file, it will be re-read the next time the script wakes up and polls for input. So you do not need to stop and restart the job. +=== Setting up environment + +The following simple setup will allow you to cross check easily source and target servers. +Assume we are in a directory: `/some/path/p4transfer` + + export P4CONFIG=.p4config + mkdir source target + cd source + vi .p4config + +and create appropriate values as per your config file for the source server e.g.: + + cat .p4config + P4PORT=source-server:1666 + P4USER=p4transfer + P4CLIENT=p4transfer_client + +And similarly create a file in the `target` sub-directory. + +This will allow you to quickly and easily `cd` between directories and be able to run commands against respective +source and target p4d instances. + === Configuration Options The comments in the file are mostly self-explanatory. It is important to specify the main values for the `[source]` and `[target]` sections. @@ -192,6 +221,9 @@ The comments in the file are mostly self-explanatory. It is important to specify cat transfer.yaml +The following included text may not display correctly when this .adoc file is viewed in GitHub - instead +download the PDF version of this doc, or open link:transfer.yaml[transfer.yaml] directly. + [source,yaml] ---- include::transfer.yaml[] @@ -203,7 +235,6 @@ In the `[general]` section, you can customize the `change_description_format` va Keywords in the format string are prefixed with `$`. Use `\n` for newlines. Keywords allowed are: `$sourceDescription`, `$sourceChange`, `$sourcePort`, `$sourceUser`. - Assume the source description is “Original change description”. Default format: @@ -229,7 +260,7 @@ might produce: There is an option in the configuration file to specify a change_map_file. If you set this option (default is blank), then P4Transfer will append rows to the specified CSV file showing the relationship between source and target changelists, and will automatically check that file in after every process. - change_map_file = change_map.csv + change_map_file = depot/import/change_map.csv The result change map file might look something like this: @@ -245,11 +276,15 @@ src-server:1666,1235,12248 It is very straight forward to use standard tools such as grep to search this file. Because it is checked in to the target server, you can also use “p4 grep”. -== Usage +IMPORTANT: You will need to ensure that the change_map filename is properly mapped in the local workspace - thus it must +include `` and other pathname components in the path. When you have created your target workspace, run `p4 client -o` +to check the view mapping. + +== Misc Usage Notes Note that since labeling itself is not versioned no labels or tags are transferred. -=== Integration +=== Integration Recrods Branching and integrating with is implemented, as long as both source and target are within the workspace view. Otherwise, the integrate action is downgraded to an add or edit. diff --git a/doc/P4Transfer.html b/doc/P4Transfer.html index fbbf4c8..fe27f45 100644 --- a/doc/P4Transfer.html +++ b/doc/P4Transfer.html @@ -445,7 +445,7 @@

P4Transfer - Helix Core Full History Migration Tool

Perforce Professional Services

version v2021.1, -2021-02-22 +2021-04-02
Table of Contents
@@ -468,17 +468,18 @@

P4Transfer - Helix Core Full History Migration Tool

  • 4.3. Script parameters
  • 4.4. Optional Parameters
  • 4.5. Long running jobs
    • -
  • 4.6. Configuration Options +
  • 4.6. Setting up environment
    • +
  • 4.7. Configuration Options
    • -
  • 5. Usage +
  • 5. Misc Usage Notes
    • @@ -751,6 +752,9 @@

    4.1. Installing P4Transfer.py

    The minimum requirements are the modules P4Transfer.py and logutils.py

    +
    +

    If you have installed P4Python as above, then check the requirements.txt for other modules to install via pip or pip3.

    +

    4.2. Getting started

    @@ -765,11 +769,11 @@

    4.2. Getting started

    -
    python3 P4Transfer.py –-sample-config > transfer.cg
    +
    python3 P4Transfer.py –-sample-config > transfer.yaml
    -

    Then edit the resulting file.

    +

    Then edit the resulting file, paying attention to the comments.

    The password stored in P4Passwd is optional if you do not want to rely on tickets. The tool performs a login if provided with a password, so it should work with security=3 or auth_check trigger set.

    @@ -796,6 +800,10 @@

    4.3. Script parameters

    P4Transfer has various options – these are documented via the -h or --help parameters.

    +
    +

    The following text may not display properly if your are viewing this P4Transfer.adoc file in GitHub. Please refer +the the .pdf version instead, or open up help.txt file directly.

    +
    usage: P4Transfer.py [-h] [-c CONFIG] [-n] [-m MAXIMUM] [-k] [-r] [-s]
@@ -882,19 +890,57 @@ 
    4.5. Long running jobs
    
 
    This will run in the background, and poll for new changes (according to poll_interval in the config file.)
    -

    You can look at the output file for progress, e.g. (avoiding long lines of text which can be output)

    +

    You can look at the output file for progress, e.g. (avoiding long lines of text which can be output), or grep the log file:

    tail -f out1 | cut -c -140
    +
    +
    +
    grep :INFO: log-P4Transfer-*.log
    +
    +

    Note that if you edit the configuration file, it will be re-read the next time the script wakes up and polls for input. So you do not need to stop and restart the job.

    -

    4.6. Configuration Options

    +

    4.6. Setting up environment

    +
    +

    The following simple setup will allow you to cross check easily source and target servers. +Assume we are in a directory: /some/path/p4transfer

    +
    +
    +
    +
    export P4CONFIG=.p4config
+mkdir source target
+cd source
+vi .p4config
    +
    +
    +
    +

    and create appropriate values as per your config file for the source server e.g.:

    +
    +
    +
    +
    cat .p4config
+P4PORT=source-server:1666
+P4USER=p4transfer
+P4CLIENT=p4transfer_client
    +
    +
    +
    +

    And similarly create a file in the target sub-directory.

    +
    +
    +

    This will allow you to quickly and easily cd between directories and be able to run commands against respective +source and target p4d instances.

    +
    +
    +
    +

    4.7. Configuration Options

    The comments in the file are mostly self-explanatory. It is important to specify the main values for the [source] and [target] sections.

    @@ -908,6 +954,10 @@

    4.6. Configuration Options

    cat transfer.yaml
    +
    +

    The following included text may not display correctly when this .adoc file is viewed in GitHub - instead +download the PDF version of this doc, or open transfer.yaml directly.

    +
    # Save this output to a file to e.g. transfer.yaml and edit it for your configuration
@@ -949,7 +999,7 @@ 
    4.6. Configuration Options
    
 poll_interval: 60
 
 # change_batch_size (Integer): changelists are processed in batches of this size
-change_batch_size: 20000
+change_batch_size: 1000
 
 # The following *_interval values result in reports, but only if mail_* values are specified
 # report_interval (Integer): Interval (in minutes) between regular update emails being sent
@@ -992,7 +1042,8 @@ 
    4.6. Configuration Options
    
 superuser: y
 
 source:
-    # P4PORT to connect to, e.g. some-server:1666
+    # P4PORT to connect to, e.g. some-server:1666 - if this is on localhost and you just
+    # want to specify port number, then use quotes: "1666"
   p4port:
     # P4USER to use
   p4user:
@@ -1000,12 +1051,14 @@ 
    4.6. Configuration Options
    
   p4client:
     # P4PASSWD for the user - valid password. If blank then no login performed.
     # Recommended to make sure user is in a group with a long password timeout!.
+    # Make sure your P4TICKETS file is correctly found in the environment
   p4passwd:
-    # P4CHARSET to use, e.g. none, utf8, etc
+    # P4CHARSET to use, e.g. none, utf8, etc - leave blank for non-unicode p4d instance
   p4charset:
 
 target:
-    # P4PORT to connect to, e.g. some-server:1666
+    # P4PORT to connect to, e.g. some-server:1666 - if this is on localhost and you just
+    # want to specify port number, then use quotes: "1666"
   p4port:
     # P4USER to use
   p4user:
@@ -1013,13 +1066,15 @@ 
    4.6. Configuration Options
    
   p4client:
     # P4PASSWD for the user - valid password. If blank then no login performed.
     # Recommended to make sure user is in a group with a long password timeout!
+    # Make sure your P4TICKETS file is correctly found in the environment
   p4passwd:
-    # P4CHARSET to use, e.g. none, utf8, etc
+    # P4CHARSET to use, e.g. none, utf8, etc - leave blank for non-unicode p4d instance
   p4charset:
 
 # workspace_root: Root directory to use for both client workspaces.
 #    This will be used to update the client workspace Root: field for both source/target workspaces
 #    They must be the same.
+#    Make sure there is enough space to hold the largest single changelist that will be transferred!
 workspace_root: /work/transfer
 
 # views: An array of source/target view mappings
@@ -1066,7 +1121,7 @@ 
    4.6. Configuration Options
    -

    4.6.1. Changelist comment formatting

    +

    4.7.1. Changelist comment formatting

    In the [general] section, you can customize the change_description_format value to decide how transferred change descriptions are formatted.

    @@ -1116,13 +1171,13 @@

    4.6.1. Changelist comment formatting

    -

    4.6.2. Recording a change list mapping file

    +

    4.7.2. Recording a change list mapping file

    There is an option in the configuration file to specify a change_map_file. If you set this option (default is blank), then P4Transfer will append rows to the specified CSV file showing the relationship between source and target changelists, and will automatically check that file in after every process.

    -
    change_map_file = change_map.csv
    +
    change_map_file = depot/import/change_map.csv
    @@ -1142,18 +1197,32 @@

    4.6.2. Recording a change list ma

    It is very straight forward to use standard tools such as grep to search this file. Because it is checked in to the target server, you can also use “p4 grep”.

    +
    + + + + + +
    + + +You will need to ensure that the change_map filename is properly mapped in the local workspace - thus it must +include <depot> and other pathname components in the path. When you have created your target workspace, run p4 client -o +to check the view mapping. +
    +

    -

    5. Usage

    +

    5. Misc Usage Notes

    Note that since labeling itself is not versioned no labels or tags are transferred.

    -

    5.1. Integration

    +

    5.1. Integration Recrods

    Branching and integrating with is implemented, as long as both source and target are within the workspace view. Otherwise, the integrate action is downgraded to an add or edit.

    @@ -1322,7 +1391,7 @@

    7.3. Running all tests

    diff --git a/doc/P4Transfer.pdf b/doc/P4Transfer.pdf index c802726..5ee1ca2 100644 Binary files a/doc/P4Transfer.pdf and b/doc/P4Transfer.pdf differ diff --git a/doc/transfer.yaml b/doc/transfer.yaml index 818a1e0..78fa952 100644 --- a/doc/transfer.yaml +++ b/doc/transfer.yaml @@ -38,7 +38,7 @@ sleep_on_error_interval: 60 poll_interval: 60 # change_batch_size (Integer): changelists are processed in batches of this size -change_batch_size: 20000 +change_batch_size: 1000 # The following *_interval values result in reports, but only if mail_* values are specified # report_interval (Integer): Interval (in minutes) between regular update emails being sent @@ -81,7 +81,8 @@ change_map_file: superuser: y source: - # P4PORT to connect to, e.g. some-server:1666 + # P4PORT to connect to, e.g. some-server:1666 - if this is on localhost and you just + # want to specify port number, then use quotes: "1666" p4port: # P4USER to use p4user: @@ -89,12 +90,14 @@ source: p4client: # P4PASSWD for the user - valid password. If blank then no login performed. # Recommended to make sure user is in a group with a long password timeout!. + # Make sure your P4TICKETS file is correctly found in the environment p4passwd: - # P4CHARSET to use, e.g. none, utf8, etc + # P4CHARSET to use, e.g. none, utf8, etc - leave blank for non-unicode p4d instance p4charset: target: - # P4PORT to connect to, e.g. some-server:1666 + # P4PORT to connect to, e.g. some-server:1666 - if this is on localhost and you just + # want to specify port number, then use quotes: "1666" p4port: # P4USER to use p4user: @@ -102,13 +105,15 @@ target: p4client: # P4PASSWD for the user - valid password. If blank then no login performed. # Recommended to make sure user is in a group with a long password timeout! + # Make sure your P4TICKETS file is correctly found in the environment p4passwd: - # P4CHARSET to use, e.g. none, utf8, etc + # P4CHARSET to use, e.g. none, utf8, etc - leave blank for non-unicode p4d instance p4charset: # workspace_root: Root directory to use for both client workspaces. # This will be used to update the client workspace Root: field for both source/target workspaces # They must be the same. +# Make sure there is enough space to hold the largest single changelist that will be transferred! workspace_root: /work/transfer # views: An array of source/target view mappings