[Info-vax] strange backup behavior full command procedure
B Hobbs
bdhobbs18 at acm.org
Mon Aug 10 13:28:43 EDT 2009
This is the full command procedure as of now. The record_stats is a
program that writes bits of job info into a database for later
analysis.
$ out := write sys$output
$ out "== ", f$environment("depth"), " == ", f$environment
("procedure"), " =="
$!
$ default = f$environment("procedure")
$ name = f$parse(default,,, "name")
$ default = f$parse(default,,, "device") + f$parse(default,,,
"directory")
$ set default 'default'
$ show default
$!
$ show time
$ out ""
$!
$ show symbol p%
$ out ""
$!
$!!$ show entry/full 'f$getqui
("display_entry","entry_number","*","this_job")'
$ show entry/full
$ out ""
$!
$!
$!! Set various date-time symbols.
$ @dua1:[users.hobbs.com]datetime.com
$ today_weekday_ddd = f$extract(0,3,today_weekday)
$ show symbol today_weekday_ddd
$!
$!
$!! Show the time during set verify.
$ set prefix "!%T "
$ set verify
$!
$!! Submit myself for tomorrow, after the 04:00 database reset.
$ submit -
/after="''tomorrow_adate'+04:10" -
/keep -
/log_file=sys$disk:[] -
/notify -
/noprinter -
/queue='f$getqui("display_entry","queue_name","*","this_job")' -
'name'
$!
$ show entry/full '$entry'
$ out ""
$!
$!
$ set noon !##
$!! Set up statistics recording.
$ define stats_database dua1:[users.hobbs.project.stats]stats
$ record_stats = "$" + f$parse("rbas-stats.exe","stats_database")
$!
$ record_stats "|''name'|''today_cdate'|job||timestamp|begin|''f$cvtime
()'"
$ record_stats "|''name'|''today_cdate'|job||char|
weekday|''today_weekday_ddd'"
$!
$!
$ backup_device = "mua0:"
$!
$!! Some of the commands need privileges.
$ set process/privileges=(log_io,phy_io,sysprv)
$!
$!! Increase buffers and blocks.
$ set rms_default -
/block_count=127 -
/buffer_count=255 -
/indexed -
/magtape -
/relative -
/sequential
$ show rms_default
$!
$!
$!! A new backup journal once a week, the day is not critical.
$ if today_weekday_ddd .eqs. "Wed" then create/log [.evidence]
backup.bjl
$!
$!
$!! Determine when to stop running.
$!! Already found out that aborting this job during a backup step does
not work well ...
$!! ... had to restart the system to deallocate the tape drive.
$!! If the job needs to stop, do it between backup steps.
$ do_not_start_after = f$cvtime("''today_adate'+08:00:00") ! default
stop time
$!
$!
$ _backup = "backup"
$ _backup = _backup + "/noalias"
$ _backup = _backup + "/brief"
$ _backup = _backup + "/ignore=(interlock)"
$ _backup = _backup + "/log"
$!
$ _backup_output = ""
$ _backup_output = _backup_output + "/block_size=65024"
$ _backup_output = _backup_output + "/norewind"
$ _backup_output = _backup_output + "/save_set"
$!
$ rmu_backup = "rmu/backup"
$ rmu_backup = rmu_backup + "/acl"
$ rmu_backup = rmu_backup + "/checksum_verification"
$ rmu_backup = rmu_backup + "/log"
$ rmu_backup = rmu_backup + "/online"
$ rmu_backup = rmu_backup + "/page_buffers=5"
$ rmu_backup = rmu_backup + "/quiet_point"
$!
$ rmu_backup_output = ""
$ rmu_backup_output = rmu_backup_output + "/accept_label"
$ rmu_backup_output = rmu_backup_output + "/active_io=5"
$ rmu_backup_output = rmu_backup_output + "/block_size=65024"
$ rmu_backup_output = rmu_backup_output + "/norewind"
$!
$!
$!! The databases are in descending order for full backup file size.
$!! Each database item is separated by a semicolon.
$!! Each database item contains:
$!! the Insyst project, a comma,
$!! the database specification, a comma,
$!! the short name, a comma,
$!! if there is a journal, then the letter j, else nothing.
$!! The list must end with a semicolon.
$ db_list = ""
$ db_list = db_list + "e1,mental_health_data_rdbfile,e1-mhs,j;"
$ db_list = db_list + "e2,mental_health_data_rdbfile,e2-mhs,j;"
$ db_list = db_list + "e1,client_service_info_rdbfile,e1-csi,;"
$ db_list = db_list + "e2,caloms_data_rdbfile,e2-caloms,;"
$!
$!
$!! The different types of operations.
$ type_full = "full"
$ type_incremental = "incr"
$ type_read = "read"
$ type_noread = "noread"
$ type_skip = "skip"
$!
$!! All operations - initialize to skip.
$ analyze_type = type_skip
$ backup_type = type_skip
$ rmu_backup_type = type_skip
$ rmu_verify_type = type_skip
$!
$!
$!! Default to incremental backups.
$!! The following configuration takes about 2.5 hours:
$!! $ analyze_type = type_read
$!! $ backup_type = type_incremental
$!! $ rmu_backup_type = type_incremental
$!! $ rmu_verify_type = type_skip
$ analyze_type = type_read
$ backup_type = type_incremental
$!!$ rmu_backup_type = type_incremental
$ rmu_backup_type = type_full !!##
$!! Hmmm, rmu/verify/incre e1-mhs began at 04:11 and was still
verifying e1-mhs at 08:40.
$!!$ rmu_verify_type = type_incremental
$!
$ if today_weekday_ddd .eqs. "Sat" .or. f$edit(p1,"lowercase") .eqs.
"full"
$ then
$!! Do full and complete backups.
$ backup_type = type_full
$ rmu_backup_type = type_full
$ if today_weekday_ddd .eqs. "Sat" then do_not_start_after = f
$cvtime("''tomorrow_adate'+03:00:00")
$ endif
$!
$ if today_weekday_ddd .eqs. "Sun"
$ then
$!! Full backups were done yesterday, and adding to the same tape
seems to slow things way down.
$!! So skip the database and disk backups and give database verify the
time.
$ analyze_type = type_skip
$ backup_type = type_skip
$ rmu_backup_type = type_skip
$!! Do full database verify.
$ rmu_verify_type = type_full !!##
$ do_not_start_after = f$cvtime("''tomorrow_adate'+03:00:00")
$ endif
$!
$!
$ show device/full 'backup_device'
$ allocate/log 'backup_device'
$ mount/noassist/foreign/nounload 'backup_device'
$!! Move past previous backup files.
$ set magtape/log/skip=end_of_tape 'backup_device'
$ show device/full 'backup_device'
$!
$!
$!! Tape labels are zero to six characters.
$ backup_device_volnam = f$getdvi(backup_device,"volnam")
$ _backup_output = _backup_output + "/label=(''backup_device_volnam')"
$ rmu_backup_output = rmu_backup_output + "/label=
(''backup_device_volnam')"
$ record_stats "|''name'|''today_cdate'|job||char|
volnam|''backup_device_volnam'"
$!
$!
$ file_extension = ".aaa"
$ gosub marker_files
$!
$!
$!!$ set noon !##
$!
$ gosub_label = "database_backup"
$ gosub process_db_list
$!
$!
$!! Step thru the disks.
$!! The name format is ddcu:
$!! dd for device type (du for disk),
$!! c for controller (a is the first controller),
$!! u for unit number (0 is the first unit).
$ next_device_name:
$ device_name = f$device("*$du*","disk")
$ out " device_name <<", device_name, ">>"
$ out " f$cvtime() <<", f$cvtime(), ">> do_not_start_after <<",
do_not_start_after, ">>"
$ if device_name .nes. "" .and. f$cvtime() .lts.
do_not_start_after
$ then
$ device_name = f$element(1,"$",device_name) !! Remove the
node name.
$ device_name = device_name - ":" !! Remove
trailing :.
$ device_name = f$edit(device_name,"lowercase") !! I prefer
lowercase.
$!
$!
$ record_stats "|''name'|''today_cdate'|''device_name'|analyze/
disk ''analyze_type'|timestamp|begin|''f$cvtime()'"
$!
$!! Analyze files, attempt to read each file.
$ if analyze_type .eqs. type_read then analyze/
disk_structure 'device_name': /read_check
$!! Analyze files, don't read files.
$ if analyze_type .eqs. type_noread then analyze/
disk_structure 'device_name':
$!
$ record_stats "|''name'|''today_cdate'|''device_name'|analyze/
disk ''analyze_type'|timestamp|end|''f$cvtime()'"
$ out ""
$ out ""
$!
$!
$ record_stats "|''name'|''today_cdate'|''device_name'|backup
''backup_type'|timestamp|begin|''f$cvtime()'"
$!
$!! Show the date/time of the last backup/record, which should be the
last full backup.
$!! The backup.sys file was created when the disk was initialized,
$!! it is not currently used by VMS and is empty.
$ directory/date=backup/noheading/notrailing/width=(filename=30)
'device_name':[000000]backup.sys
$!
$ backup_exclude = ""
$!! If there are Insyst database files on this disk,
$!! exclude them from the backup.
$!! From insyst_backup.com.
$ exclude_spec = "[000000.e%.*.database]*.*"
$ if f$search("''device_name':''exclude_spec'") .nes. "" -
then backup_exclude = backup_exclude + "," + exclude_spec
$ exclude_spec = "[000000.insyst_e%.*.database]*.*"
$ if f$search("''device_name':''exclude_spec'") .nes. "" -
then backup_exclude = backup_exclude + "," + exclude_spec
$ exclude_spec = "[000000.insyst_e%%.*.database]*.*"
$ if f$search("''device_name':''exclude_spec'") .nes. "" -
then backup_exclude = backup_exclude + "," + exclude_spec
$ if backup_exclude .nes. ""
$ then
$ backup_exclude = backup_exclude - "," ! remove the leading
comma
$ backup_exclude = "/exclude=(" + backup_exclude + ")" ! add
qualifier and parens
$ endif
$ show symbol backup_exclude
$!
$ if backup_type .eqs. type_full
$ then
$!! Backup all (well, most) files on this disk.
$!! Didn't use /image because it ignores /exclude.
$ verify_state = f$verify(1)
$ '_backup' -
/fast -
/journal=[.evidence]backup.bjl -
/list=[.evidence]'device_name'.lis -
/record -
'device_name':[000000...]*.*;* -
'backup_exclude' -
'backup_device''today_weekday_ddd'-'device_name'.bck -
'_backup_output'
$ show symbol $status
$ verify_state = f$verify(verify_state)
$ endif
$!
$ if backup_type .eqs. type_incremental
$ then
$!! Backup files that have changed since the last full backup.
$!! Didn't put /record on the incremental
$!! so we can see how many files accumulate until the next full
backup.
$ verify_state = f$verify(1)
$ '_backup' -
/journal=[.evidence]backup.bjl -
/list=[.evidence]'device_name'.lis -
'device_name':[000000...]*.*;* -
'backup_exclude' -
/since=backup -
'backup_device''today_weekday_ddd'-'device_name'.bck -
'_backup_output'
$ show symbol $status
$ verify_state = f$verify(verify_state)
$ endif
$!
$ backup_exclude = ""
$!
$ record_stats "|''name'|''today_cdate'|''device_name'|backup
''backup_type'|timestamp|end|''f$cvtime()'"
$ out ""
$ out ""
$!
$!
$ goto next_device_name
$ endif
$!
$!
$!! Backup the command procedure, log, and other assorted files.
$!! No journal or list options, so most files are properly backed up.
$ record_stats "|''name'|''today_cdate'|evidence||timestamp|begin|''f
$cvtime()'"
$ verify_state = f$verify(1)
$ '_backup' -
[...]*.*;* -
'backup_device''today_weekday_ddd'-evidence.bck -
'_backup_output'
$ purge/before="''today_adate'-45-"/log []'name'.log, [.evidence]
*.*
$ verify_state = f$verify(verify_state)
$ record_stats "|''name'|''today_cdate'|evidence||timestamp|end|''f
$cvtime()'"
$!!$ set on !##
$!
$!
$ file_extension = ".zzz"
$ gosub marker_files
$!
$!
$!! What has accumulated on this tape?
$ dismount/nounload 'backup_device'
$ mount/noassist/nounload 'backup_device' 'backup_device_volnam'
$ directory/date=(created)/size=used/width=(filename=20,size=10)
'backup_device'
$!
$!
$ dismount/nounload 'backup_device'
$ deallocate 'backup_device'
$ show device/full 'backup_device'
$!
$!
$!! Rmu/verify takes a looooong time to process a database.
$!! Perform rmu/verify after all backups complete
$!! so the job can be terminated without skipping backups
$!! or locking up the tape drive.
$ out " rmu_verify_type <<", rmu_verify_type, ">>"
$ if rmu_verify_type .nes. type_skip
$ then
$ gosub_label = "database_verify"
$ gosub process_db_list
$ endif
$!
$!
$ record_stats "|''name'|''today_cdate'|job||timestamp|end|''f$cvtime
()'"
$ set on !##
$!
$!
$ exit
$ database_backup:
$ record_stats "|''name'|''today_cdate'|''short_name'|rmu/backup
''rmu_backup_type'|timestamp|begin|''f$cvtime()'"
$!
$ if rmu_backup_type .eqs. type_full
$ then
$!! Full database backup.
$ verify_state = f$verify(1)
$ 'rmu_backup' /noincremental -
/journal=[.evidence]'short_name'.jnl -
/restore_options=[.evidence]'short_name'.opt -
'database_spec' -
'backup_device''today_weekday_ddd'-'short_name'.rbf -
'rmu_backup_output'
$ verify_state = f$verify(verify_state)
$ endif
$!
$ if rmu_backup_type .eqs. type_incremental
$ then
$!! Incremental database backup.
$ verify_state = f$verify(1)
$ 'rmu_backup' /incremental=complete -
/journal=[.evidence]'short_name'.jnl -
/restore_options=[.evidence]'short_name'.opt -
'database_spec' -
'backup_device''today_weekday_ddd'-'short_name'.rbf -
'rmu_backup_output'
$ verify_state = f$verify(verify_state)
$ endif
$!
$ record_stats "|''name'|''today_cdate'|''short_name'|rmu/backup
''rmu_backup_type'|timestamp|end|''f$cvtime()'"
$ out ""
$ out ""
$!
$!
$ if journal_flag .eqs. "j"
$ then
$!! Database journal backup.
$ record_stats "|''name'|''today_cdate'|''short_name'-j|backup|
timestamp|begin|''f$cvtime()'"
$!
$!! The dates will show what time period these journals cover.
$ directory/date=(created,modified)/size=all/width=
(filename=50,size=10) insyst_data_journals
$!
$ verify_state = f$verify(1)
$ '_backup' -
/journal=[.evidence]backup.bjl -
/list=[.evidence]'short_name'-j.lis -
insyst_data_journals -
'backup_device''today_weekday_ddd'-'short_name'-j.bck -
'_backup_output'
$ show symbol $status
$ verify_state = f$verify(verify_state)
$!
$!! Don't want to use this command at this time,
$!! may cause the journal file to switch.
$!! Database journal backup.
$!!$ rmu/backup/after_journal -
$!! /log -
$!! /quiet_point -
$!! 'database_spec' -
$!! 'backup_device''today_weekday_ddd'-'short_name'-j.aij
$!!$ show symbol $status
$!!$ show symbol rdm$aij_*
$!
$!! It appears that the Insyst backup script did a backup/delete of
the database journal files.
$!! Since this script replaces the Insyst backup script,
$!! I suppose it should also manage the journals.
$!! Keep a bit over two weeks of journal files
$!! so there is a complete set even if one weekly full backup is
missed.
$!! Use /before instead of /keep in case the database is restarted ...
multiple times.
$ purge/before="''today_adate'-15-"/log insyst_data_journals
$!
$ record_stats "|''name'|''today_cdate'|''short_name'-j|backup|
timestamp|end|''f$cvtime()'"
$ out ""
$ out ""
$ endif !! if f$element(1,"-",short_name) .eqs. "mhs"
$ return
$ database_verify:
$ record_stats "|''name'|''today_cdate'|''short_name'|rmu/verify
''rmu_verify_type'|timestamp|begin|''f$cvtime()'"
$!
$ if rmu_verify_type .eqs. type_full
$ then
$!! Full database verify.
$ verify_state = f$verify(1)
$ rmu/verify/all/log/transaction_type=read_only 'database_spec'
$ show symbol $status
$ verify_state = f$verify(verify_state)
$ endif
$!
$ if rmu_verify_type .eqs. type_incremental
$ then
$!! Incremental database verify.
$ verify_state = f$verify(1)
$ rmu/verify/all/log/transaction_type=read_only 'database_spec' /
incremental
$ show symbol $status
$ verify_state = f$verify(verify_state)
$ endif
$!
$ record_stats "|''name'|''today_cdate'|''short_name'|rmu/verify
''rmu_verify_type'|timestamp|end|''f$cvtime()'"
$ out ""
$ out ""
$ return
$ insyst_project_setup:
$ verify_state = f$verify(0,0)
$!! Switch to the proper project.
$!! Symbols e1 and e2 don't work for user backup.
$ @insyst$project$root1$gbl:[misc.installed]
insyst_project_setup.com 'insyst_project'
$ verify_state = f$verify(verify_state)
$ out ""
$ out ""
$ return
$ marker_files:
$!! If we're skipping all backups, then don't bother with the marker
files.
$ if rmu_backup_type .eqs. type_skip .and. backup_type .eqs.
type_skip then return
$!
$!! The file contents are not important,
$!! the file name shows whether the files are full or incremental
backups.
$!! Tried to use create to make empty files,
$!! but it caused backup to generate position errors.
$ record_stats "|''name'|''today_cdate'|marker|''file_extension'|
timestamp|begin|''f$cvtime()'"
$ verify_state = f$verify(1)
$ '_backup' -
[]'name'.com; -
'backup_device''today_weekday_ddd'-
db-'rmu_backup_type''file_extension' -
'_backup_output'
$ '_backup' -
[]'name'.com; -
'backup_device''today_weekday_ddd'-
disk-'backup_type''file_extension' -
'_backup_output'
$ verify_state = f$verify(verify_state)
$ record_stats "|''name'|''today_cdate'|marker|''file_extension'|
timestamp|end|''f$cvtime()'"
$ return
$ process_db_list:
$ element = 0
$ next_database:
$ insyst_project = f$element(element,";",db_list)
$ out " insyst_project <<", insyst_project, ">>"
$ out " f$cvtime() <<", f$cvtime(), ">> do_not_start_after
<<", do_not_start_after, ">>"
$ if insyst_project .nes. "" .and. f$cvtime() .lts.
do_not_start_after
$ then
$ database_spec = f$element(1,",",insyst_project)
$ short_name = f$element(2,",",insyst_project)
$ journal_flag = f$element(3,",",insyst_project)
$ insyst_project = f$element(0,",",insyst_project)
$!
$ gosub insyst_project_setup
$ gosub 'gosub_label'
$!
$ element = element + 1
$ goto next_database
$ endif
$!
$ insyst_project = "e0"
$ gosub insyst_project_setup
$ return
More information about the Info-vax
mailing list