D' J d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl Z ddl
Z
ddl m Z ddl
mZ dd lmZ dd
lmZ dd lmZ dd lmZ dd
lmZ ddlmZ ddlmZ ddlm Z ddlm Z ddlm Z ddlm Z ddl m!Z! ddl m"Z" ddlm#Z# e jH Z%dZ&de&z dz Z'd e#jP z dz d z e#jP z d z d z d z d z d!z e#jR z d"z d#z d$z d%z d!z e#jR z d&z d'z Z* G d( d)e Z+y)*zEImplementation of hash command for calculating hashes of local files. )absolute_import)print_function)division)unicode_literalsN)Command)CommandArgument) ApiSelector)CommandException)FileProgressCallbackHandler)ProgressCallbackWithTimeout)StorageUrlFromString) FileMessage) FinalMessage) boto_util) constants)hashing_helper)parallelism_framework_util)GcloudStorageFlag)GcloudStorageMap) shim_utilz*
gsutil hash [-c] [-h] [-m] filename...
z
SYNOPSIS
a
DESCRIPTION
Calculate hashes on local files, which can be used to compare with
``gsutil ls -L`` output. If a specific hash option is not provided, this
command calculates all gsutil-supported hashes for the files.
Note that gsutil automatically performs hash validation when uploading or
downloading files, so this command is only needed if you want to write a
script that separately checks the hash.
If you calculate a CRC32c hash for files without a precompiled crcmod
installation, hashing will be very slow. See "gsutil help crcmod" for details.
OPTIONS
-c Calculate a CRC32c hash for the specified files.
-h Output hashes in hex format. By default, gsutil uses base64.
-m Calculate a MD5 hash for the specified files.
z>--format=value[separator="",terminator=""](digest_format.sub("z", "Hashes ["),z url.sub("z ", "] for ").sub("$", ":^\^n"),z,md5_hash.yesno(yes=" Hash (md5): ", no=""),zmd5_hash.yesno(no=""),z$md5_hash.yesno(yes="NEWLINE", no="")z.sub("NEWLINE", "z"),z2crc32c_hash.yesno(yes=" Hash (crc32c): ", no=""),zcrc32c_hash.yesno(no=""),z'crc32c_hash.yesno(yes="NEWLINE", no="")z"))c
e Zd ZdZ ej
dg edej dddde j ge j e j g Z
ej dd gd
d ei Z fd
Zed Zd Zd Z xZS ) HashCommandz&Implementation of gsutil hash command.hash chmTFr ) command_name_aliasesusage_synopsismin_argsmax_argssupported_sub_args file_url_okprovider_url_okurls_start_arggs_api_supportgs_default_apiargparse_argumentschecksum command_helpzCalculate file hashes) help_namehelp_name_aliases help_typehelp_one_line_summary help_textsubcommand_help_textc T t ddt gt d d d d }t | j | j
D cg c] \ }}| c}}z }d|v rd|vr|xj dgz
c_ n d|v rd|vr|xj d gz
c_ t | ! | S c c}}w )
Nstorager z--hex)-h-c-m)gcloud_commandflag_mapr3 r4 z
--skip-md5z
--skip-crc32c) r _GCLOUD_FORMAT_STRINGr setargssub_optsr5 superget_gcloud_storage_args)selfgcloud_storage_mapflag_args_set __class__s &platform/gsutil/gslib/commands/hash.pyr< z#HashCommand.get_gcloud_storage_argst s ) !
$G,
499DMM BMqM BBCH xD 0''L>9'
d( 2''O+<<' 7 *+= >>
Cs B$ c d}d}d }d }d}d}|r,|D ]' \ } }
| dk( rd}d}| dk( r d}d }d
} | d k( s$d}d}) |sd}d}|r3t j s |j t j |||||fS ) aZ Returns behavior variables based on input options.
Args:
sub_opts: getopt sub-arguments for the command.
logger: logging.Logger for the command.
Returns:
Tuple of
calc_crc32c: Boolean, if True, command should calculate a CRC32c checksum.
calc_md5: Boolean, if True, command should calculate an MD5 hash.
format_func: Function used for formatting the hash in the desired format.
cloud_format_func: Function used for formatting the hash in the desired
format.
output_format: String describing the hash output format.
Fc H t j | j S N)r Base64EncodeHash hexdigestdigests rC z(HashCommand._ParseOpts.. s ~66v7G7G7IJ c | S rF rI s rC rK z(HashCommand._ParseOpts.. s vrL base64r3 Tr2 hexc " | j S rF )rH rI s rC rK z(HashCommand._ParseOpts.. s
v'7'7'9rL c J t j | j d S )Nascii)r Base64ToHexHashdecoderI s rC rK z(HashCommand._ParseOpts.. s ,,V4;;GDrL r4 )r UsingCrcmodExtensionwarnr SLOW_CRCMOD_WARNING) clsr: logger calc_crc32ccalc_md5 format_funccloud_format_funcfound_hash_option
output_formatounused_as rC
_ParseOptszHashCommand._ParseOpts s " KHJ - M!+!X 9 + "
$Y -9+
$Y( "
" kh999; kk.445 +/@- OOrL c i }|r"t j j d |d< |rt j |d< |S )a4 Constructs the dictionary of hashes to compute based on the arguments.
Args:
calc_crc32c: If True, CRC32c should be included.
calc_md5: If True, MD5 should be included.
Returns:
Dictionary of {string: hash digester}, where string the name of the
digester algorithm.
zcrc-32ccrc32cmd5)crcmod
predefinedCrcr GetMd5)r= r[ r\ hash_dicts rC _GetHashClassesFromArgsz#HashCommand._GetHashClassesFromArgs sC I " - - 1 1) jA d | |r|j8 |
d < |r|j: |
d
< tC d|d|
d tE jF |
D ]4 \ }}tC d|d| j r || n || 6 i |s tI d tK | j j tM t# j" y# 1 sw Y xY w)z)Command entry point for the hash command.F)re customerEncryptionmd5Hashsize)bucket_listing_fieldsTN)rp finished message_typeHashing)src_urloperation_namerb)callback_processorzNo hashes present for %srf re zHashes [z] for :z Hash (z): zNo files matchedr )'rc r: rZ r9 WildcardIterator IterObjectsr
storage_url object_name IsFileUrlospathgetsize
gsutil_api status_queueputr time FILE_HASHr r callrl openr CalculateHashesFromContents root_objectrp ro re logging getLoggerrW printsix iteritemsr
_PutToQueueWithTimeoutr )r= r[ r\ r] r^ r` matched_oneurl_strfile_refurl file_name file_sizerx rk fp obj_metadata md5_presentcrc32c_presentnamerJ s rC
RunCommandzHashCommand.RunCommand s t}}dkkB[(K):K99++G4@@! A
( "7+ ((44 ( 2 2 4ggooi0)
//
&
&
*
*# ))+ (#('2'<'< > ? ;)$//*F*F2Fw2O9BDDHD J
22;I)It$ 6 6I2D
F % //
&
&
*
*# ))+ (#''2'<'< > ? )!--,"'')$,,D8+'..d:. ^ $ $%? I
+ 3 3Ie
"."5"5Ih
}i@A MM) 4LD&
@C
V)<.?.G*HJ K 5c
n / 004??77' 46 C %$s 3LL" )__name__
__module__ __qualname____doc__r CreateCommandSpec _SYNOPSISr NO_MAXr JSONr MakeZeroOrMoreFileURLsArgument command_specHelpSpec_DETAILED_HELP_TEXT help_specr< classmethodrc rl r
__classcell__)rB s @rC r r Y s . +** !&&' %%H/HHJK M, g # 3#
)?* .P .P`$A
rL r ),r
__future__r r r r r r r rg r
gslib.commandr gslib.command_argumentr gslib.cs_api_mapr gslib.exceptionr
gslib.progress_callbackr r gslib.storage_urlr
gslib.thread_messager r gslib.utilsr r r r gslib.utils.shim_utilr r r PutToQueueWithTimeoutr r r get_format_flag_caretget_format_flag_newliner7 r rN rL rC r s L & % '
! 2 ( , ? ? 2 , - ! ! & 2 3 2 !3II
4 O#I##%&(9:
AF F <
< #LL <);;=> AEE HKK t
' t
rL