#!/bin/bash

RAINBOW_CONFIG_LOCATION="/opt/etc/rainbow.conf"
RAINBOW_CLEANUP_SCRIPT="rainbow_cleanup.sh"

[ -e "$RAINBOW_CONFIG_LOCATION" ] || exit 1
source $RAINBOW_CONFIG_LOCATION

if [ -d "${RAINBOW_CONFIG_LOCATION}.d" ]; then
    for confd in $( find "${RAINBOW_CONFIG_LOCATION}.d" -name '*.conf' ); do
        source $confd
    done
fi

rainbow_cleanup () {
    trap '' SIGINT SIGTERM
    [ -e "$RAINBOW_CLEANUP_SCRIPT" ] && source "$RAINBOW_CLEANUP_SCRIPT"
    exit
}

trap rainbow_cleanup SIGINT SIGTERM EXIT

# defaults

CLOUD_USERDATA_DIR="userdata"
CLOUD_ISODISK_NAME="data.iso"
CLOUD_DATADISK_NAME="data.img"
CLOUD_INFILES_LIST="inputfiles.list"
CLOUD_OUTFILES_LIST="outputfiles.list"
CLOUD_ENV_FILE="job_environment.sh"

OUTFILES_METHOD=${CLOUD_OUTFILES_MOTOD:-exact} 
OUTFILES_DISK=${OUTFILES_DISK:-data} 
OUTFILES_LOCATION=${OUTFILES_LOCATION:-/}
LRMS_EXTRA_MEMORY=${LRMS_EXTRA_MEMORY:-1024}
RAINBOW_DEFAULT_WALLTIME=${RAINBOW_DEFAULT_WALLTIME:-43200}

# clean-up functions 

init_cleanup_script () {
    echo "source $RAINBOW_CONFIG_LOCATION" > $RAINBOW_CLEANUP_SCRIPT
    echo "source $CLOUD_LIBEXEC_LOCATION/nethelper-client.sh" >> $RAINBOW_CLEANUP_SCRIPT
}

add2_cleanup_script () {
    cat >> $RAINBOW_CLEANUP_SCRIPT
    echo >> $RAINBOW_CLEANUP_SCRIPT
}

# generator functions 

mkpasswd() {
    local maxsize=$1
    local MAXSIZE=${maxsize:-8}
    local array1=(
        q w e r t y u i o p a s d f g h j k l z x c v b n m Q W E R T Y U I O P A S D
        F G H J K L Z X C V B N M 1 2 3 4 5 6 7 8 9 0 \@ \# \$ \& \* 
    )
    local MODNUM=${#array1[*]}
    local pwd_len=0
    while [ $pwd_len -lt $MAXSIZE ]
    do
        index=$(($RANDOM%$MODNUM))
        echo -n "${array1[$index]}"
        ((pwd_len++))
    done
}

# information output functions

check_push_info_needed () {
    [ -n "${GH_CONNECT}" ] && push_service="gh"
    [ -n "${NOTIFY_EMAIL}" ] && push_service="mail"
    [ -n "${PILOT}" ] && push_service="${PILOT}"

    push_service_libexec="$CLOUD_LIBEXEC_LOCATION/push_info_${push_service}.sh"
    if [ -f "$push_service_libexec" ]; then 
        source "$push_service_libexec"
        [ $? -eq 0 ] && export push_info=1
    fi
}

r_debug () {
    [ -n "${RAINBOW_DEBUG}" ] && echo $@
}

echo_action () {
    echo -e "\t$@"
}

exit_plus () {
    local exit_code=$1
    local error_message=$2

    if [ -n "$push_info" -a $exit_code -ne 0 ]; then
        push_failed "$error_message"
    fi

    echo "$error_message"
    exit $exit_code
}


# Rainbow life cycle processing function
evaluate_arc_joboption_env () {
    env_idx=0
    local env_var="joboption_env_$env_idx"
    while [ -n "${!env_var}" ]; do
        eval ${!env_var}
        env_idx=$((env_idx+1))
        env_var="joboption_env_$env_idx"
    done
}

dump_arc_joboption_env () {
    local dump_env_idx=0
    local env_var="joboption_env_$dump_env_idx"
    while [ -n "${!env_var}" ]; do
        echo "${!env_var}"
        dump_env_idx=$((dump_env_idx+1))
        env_var="joboption_env_$dump_env_idx"
    done
}

add_arc_joboption_env () {
    local var_name=$1
    local var_value=$2
    [ -z "$env_idx" ] && exit_plus 1 "FATAL: There is no environment index defined. ARC joboption_env should be evaluated first."
    eval joboption_env_$env_idx="${var_name}=${var_value}"
    env_idx=$((env_idx+1))
}

requested_resources_to_arc_env () {
     # memory
     add_arc_joboption_env 'CLOUD_MEMORY' $joboption_memory
     export joboption_memory=$(( $joboption_memory + $LRMS_EXTRA_MEMORY ))
     # walltime (in seconds)
     [ -z "$joboption_walltime" ] && joboption_walltime=${RAINBOW_DEFAULT_WALLTIME}
     export joboption_walltime=$(( $joboption_walltime + 600 ))
     add_arc_joboption_env 'CLOUD_WALLTIME' $joboption_walltime
     # cpus
     [ "x$joboption_countpernode" = "x-1" ] && job_cpus=$joboption_count || job_cpus=$joboption_countpernode
     add_arc_joboption_env 'CLOUD_CPUS' $job_cpus
     # runtime stdout/stderr 
     # no stdout/stderr for arccat unless we will explicitely write to it
     add_arc_joboption_env 'RUNTIME_STDOUT' "$joboption_stdout"
     add_arc_joboption_env 'RUNTIME_STDERR' "$joboption_stderr"
}

wrap_job_environemt () {
     local executable=${joboption_args# *}
     executable=${executable% *}
     echo "JOB_EXECUTABLE=${executable}" > "$CLOUD_ENV_FILE"
     joboption_args='/bin/true'
     joboption_arg_0='/bin/true'
     dump_arc_joboption_env >> "$CLOUD_ENV_FILE"
     echo "$CLOUD_ENV_FILE" >> "$CLOUD_INFILES_LIST"
     [ "$CLOUD_DATA_METHOD" = "ISO" ] && CLOUD_DATA_METHOD="DISK"
}

create_input_files_list () {
    local files_list=$1
    local inf_idx=0
    local inf_var="joboption_inputfile_$inf_idx"
    local inf_name
    touch "$files_list"
    while [ -n "${!inf_var}" ]; do
        inf_name="${!inf_var#/}"
        inf_idx=$((inf_idx+1))
        inf_var="joboption_inputfile_$inf_idx"
        # exclude non-data files (VM system image and description)
        [ "${inf_name}" = "${CLOUD_VM_SYSIMAGE}" ]    && continue
        [ "${inf_name}" = "${CLOUD_VM_DESCRIPTION}" ] && continue
        echo "${inf_name}" >> "$files_list"
    done
}

create_output_files_list () {
    local files_list=$1
    local outf_idx=0
    local outf_var="joboption_outputfile_$outf_idx"
    echo -n > "$files_list"
    while [ -n "${!outf_var}" ]; do
        echo "${!outf_var#/}" >> "$files_list"
        outf_idx=$((outf_idx+1))
        outf_var="joboption_outputfile_$outf_idx"
    done
}

setup_local_scratch () {
    if [ -n "$RAINBOW_WN_SCRATCH" ]; then
        # create local scratch dir to speedup I/O
        export RAINBOW_SCRATCH_DIR="$( mktemp -d --tmpdir=$RAINBOW_WN_SCRATCH rainbow.scratch.XXXXXXX )/"
        # local data copy should be performed to local scratch if available
        export CLOUD_USERDATA_DIR="${RAINBOW_SCRATCH_DIR}${CLOUD_USERDATA_DIR}"
        echo "rm -rf \"${RAINBOW_SCRATCH_DIR}\"" | add2_cleanup_script
    fi
}

userdata_copy_staged_in () {
    local files_list=$1
    local target_dir=$2

    mkdir -p "${target_dir}"
    for inf_name in $( cat "$files_list" ); do
        if [ ! -e "${inf_name}" ]; then 
            echo "ERROR: Input file ${inf_name} declared but not exists."
            exit
        fi
        cp --copy-contents "${inf_name}" "${target_dir}/"
    done
}


create_vm_data_disk () {
    local disk_method=$1
    local staged_dir=$2

    if [ "${disk_method}" = "DISK" ]; then
        export VM_DISK_2="${staged_dir%/}.img"
        DATA_DISK_FS=${DATA_DISK_FS:-vfat} 
        DATA_DISK_SIZE=${DATA_DISK_SIZE:-100G}
        create_disk_img "${staged_dir}" "$VM_DISK_2" "$DATA_DISK_FS" "$DATA_DISK_SIZE" "${DATA_DISK_FORMAT}"
        export VM_DISK_2_FORMAT="${DATA_DISK_FORMAT}"
        chmod 666 $VM_DISK_2
        echo "rm -f \"${VM_DISK_2}\"" | add2_cleanup_script
    elif [ "${disk_method}" = "ISO" ]; then
        mkisofs -J -R -o "${CLOUD_ISODISK_NAME}" -V DATA "${staged_dir}/"
        export VM_CD_DISK_1="${CLOUD_ISODISK_NAME}"
        chmod 666 $VM_CD_DISK_1
        echo "rm -f \"${VM_CD_DISK_1}\"" | add2_cleanup_script
    elif [ "${disk_method}" = "VVFAT" ]; then
        # TODO: vvfat handling in case libvirt hv adapter used
        export VM_VVFAT_DISK_1="${staged_dir}"
        chmod 666 $VM_VVFAT_DISK_1
        keep_userdata_dir=1
    elif [ "${disk_method}" = "USERDISK" ]; then
        #TODO: add and test support for thin-provisioned disks
        echo_action "\tNot implemented yet"
    fi
    [ -z "$keep_userdata_dir" ] && rm -rf "${staged_dir}/"
}

extract_vm_data_disk () {
    local files_list=$1
    local disk_method=$2
    local stagein_dir=$3

    if [ $( cat ${files_list} | wc -l ) -gt 0 ]; then
        echo "Extracting output files from guest... "

        extract_dir="outfiles"
        extract_outfiles=1
        if [ "${OUTFILES_DISK}" = "data" ]; then 
            if [ "${disk_method}" = "ISO" ]; then
                echo_action "Failed. ISO input files method does not support write operations to data disk."
            elif [ "${disk_method}" = "VVFAT" ]; then
                if check_vvfat_support ; then
                    unset extract_outfiles
                    extract_dir="$stagein_dir"
                else
                    extract_img="${stagein_dir%/}.img"
                fi
            else
                extract_img="$VM_DISK_2"
            fi
        elif [ "${OUTFILES_DISK}" = "system" ]; then
            extract_img="$VM_DISK_1"
        else
            echo_action "Failed. Wrong OUTFILES_DISK specified, only 'data' and 'system' are supported."
        fi

        mkdir -p "$extract_dir"
        [ -n "$extract_outfiles" ] && extract_disk_img "$extract_img" "$extract_dir" "${OUTFILES_LOCATION}"

        if [ $? -ne 0 ]; then
            echo_action "Failed. Image extraction tool returns bad exit status."
        else 
            if [ "$OUTFILES_METHOD" = "exact" ]; then
                for outf in $( cat ${files_list} ); do
                    mv "$extract_dir/$outf" "$outf" >/dev/null 2>&1
                done
            elif [ "$OUTFILES_METHOD" = "tar" ]; then
                tar cf $( cat ${files_list} | head -n 1 ) $extract_dir
            else
                echo "Failed. Wrong OUTFILES_METHOD specified, only 'exact' and 'tar' are supported."
            fi
            echo_action "Done."
        fi
    fi
}

prepare_vm_system_disk () {
    local system_disk=$1

    chmod 755 ${PWD}
    export VM_DISK_1="${system_disk}"
    [ ! -f "$VM_DISK_1" ] && exit_plus 1 "Fatal. There is no VM image staged in." 
    # in case VM image symlinked directly from ARC datastaging cache without job-specific copy
    if [ -n "$VM_IMG_COMPRESSED" ]; then
        cat "$VM_DISK_1" | $VM_IMG_COMPRESSED -d > "${RAINBOW_SCRATCH_DIR}${VM_DISK_1%.*}.unpacked.img"
        export VM_DISK_1="${RAINBOW_SCRATCH_DIR}${VM_DISK_1%.*}.unpacked.img"
    elif [ -L "$VM_DISK_1" ]; then
        localcopy=0
        readlink -f $VM_DISK_1 | grep -q ${GRID_GLOBAL_JOBID##*/}
        [ $? -ne 0 ] && localcopy=1
        [ -w "$( readlink -f $VM_DISK_1 )" ] || localcopy=1
        if [ $localcopy -eq 1 ]; then
            cp "$VM_DISK_1" "${RAINBOW_SCRATCH_DIR}${VM_DISK_1%.*}.local.img"
            export VM_DISK_1="${RAINBOW_SCRATCH_DIR}${VM_DISK_1%.*}.local.img"
        fi
    fi
    chmod 666 $VM_DISK_1
    echo "rm -f \"${VM_DISK_1}\"" | add2_cleanup_script
}

change_vm_access_password () {
    if [ -n "$guest_img_processing" ]; then
        echo "Changing access password in the VM image..."
        [ "$RUNTIME_PASSWORD" = "dynamic" ] && RUNTIME_PASSWORD="$(mkpasswd)"
        [ "$VM_OS_TYPE" = "windows" ] && RUNTIME_PASSWORD=${RUNTIME_PASSWORD:0:14}
        change_guest_password "$VM_DISK_1" "${VM_ACCESS_USER:-user}" "$RUNTIME_PASSWORD"
        if [ $? -eq 0 ]; then
            VM_ACCESS_PASSWORD="$RUNTIME_PASSWORD"
            echo_action "Done."
        else
            echo_action "Failed. Image-specific pre-configured password will be used."
        fi
    else
        echo "WARNING: Dynamic password changing for VM image access was requested but is not supported by WN."
        echo_action "Image-specific pre-configured password will be used."
    fi
}

define_vm_params () {
    local vm_description=$1
    # source VM description file with hardware and OS description
    [ -n "$vm_description" ] && source $vm_description >/dev/null 2>&1
    # job-defined resources
    export VM_MEMORY_SIZE="${CLOUD_MEMORY}"
    export VM_CPUS="${CLOUD_CPUS}"
    export VM_WALLTIME="${CLOUD_WALLTIME}"
    export VM_NAME="${GRID_GLOBAL_JOBID##*/}"
    # pilot-defined resources
    [ -n "$PILOT_CPUS" ] && VM_CPUS="$PILOT_CPUS"
    [ -n "$PILOT_MEMORY" ] && VM_MEMORY_SIZE="$PILOT_MEMORY"
    [ -n "$PILOT_WALLTIME" ] && VM_WALLTIME="$PILOT_WALLTIME"
}

define_vm_access_method () {
    # access method should be defined in VM-image config-file. Default is RDP.
    if [ -z "$VM_ACCESS_METHOD" ]; then 
        if   [ "$VM_OS_TYPE" = "windows" ]; then VM_ACCESS_METHOD="RDP"
        elif [ "$VM_OS_TYPE" = "linux" ]  ; then VM_ACCESS_METHOD="SSH"
        else VM_ACCESS_METHOD="RDP"
        fi
    fi
    [ "${VM_ACCESS_METHOD}" = "RDP" ] && VM_LOCAL_PORT=3389
    [ "${VM_ACCESS_METHOD}" = "SSH" ] && VM_LOCAL_PORT=22
}

lease_vm_mac () {
    echo "Obtaining network configuration... "
    export VM_NET_MAC="$(get_mac $((VM_WALLTIME+3600)) )"
    if [ -n "${VM_NET_MAC}" ]; then 
        echo "free_mac \"${VM_NET_MAC}\"" | add2_cleanup_script
        echo_action "Done."
    else
        echo_action "Failed."
        exit_plus 0 "There are no free network slots to start VM."
    fi
}

start_vm () {
    local hypervisor=$1
    local vm_runscript="$CLOUD_LIBEXEC_LOCATION/hv_adapter_${hypervisor}.sh"

    echo "Starting VM on WN's $hypervisor hypervisor... "
    [ ! -f "$vm_runscript" ] && exit_plus 1 "Failed. HyperVisor adapter for \"${hypervisor}\" is not installed. Check WN configuration."

    source $vm_runscript 

    VM_RUNSTATUS=$?
    if [ $VM_RUNSTATUS -eq 0 ]; then
        echo_action "Done."
    else
        [ $VM_RUNSTATUS -eq 2 ] && echo_action "Failed to create network interface for VM."
        exit_plus 1 "Failed to create VM."
    fi
}

wait_vm_network_ready () {
    echo "Waiting for VM network becomes ready... "
    VM_NET_CHECK_INTERVAL=${VM_NET_CHECK_INTERVAL:-15}
    VM_NET_CHECK_COUNT=${VM_NET_CHECK_COUNT:-20}
    no_network_cnt=0
    while [ $no_network_cnt -lt $VM_NET_CHECK_COUNT ]; do
        sleep $VM_NET_CHECK_INTERVAL
        result=$( helper_mping "$VM_NET_MAC" )
        [ "$result" = "0" ] && break
        no_network_cnt=$((no_network_cnt+1))
    done

    if [ $no_network_cnt -lt $VM_NET_CHECK_COUNT ]; then
        echo_action "Done. Virtual machine IP ping successful."
    else
        echo_action "Failed."
        destroy_vm
        exit_plus 1 "No network connection within $((VM_NET_CHECK_COUNT*VM_NET_CHECK_INTERVAL)) seconds. Destroying the machine."
    fi
}

get_interactive_access_socket () {
    local vm_net_mac=$1
    local vm_local_port=$2
    local failed_cnt=0

    while [ $failed_cnt -lt $VM_NET_CHECK_COUNT ]; do
        sleep $VM_NET_CHECK_INTERVAL
        result=$( helper_mnc $vm_net_mac $vm_local_port )
        [ "$result" = "0" ] && break
        failed_cnt=$((failed_cnt+1))
    done

    if [ $failed_cnt -lt $VM_NET_CHECK_COUNT ]; then
        get_port_forward $vm_net_mac $vm_local_port 
    fi
}

print_interactive_access_credentials () {
cat <<END
-------------------------------------------------------------
You can now connect to VM ${VM_ACCESS_METHOD} with the following credentials:
   Address: ${VM_ACCESS_SOCKET}
   Login: ${VM_ACCESS_USERNAME:-user}
   Password: ${VM_ACCESS_PASSWORD:-N/A}

-------------------------------------------------------------
END
}

monitor_vm_is_running () {
    local vm_walltime=$1
    while [ $vm_walltime -gt 0 ]; do
        sleep 10
        if ! check_vm_running; then
            echo "VM is no longer running (shutted down by operator or failed). Exiting walltime sleep."
            break
        fi
        vm_walltime=$((vm_walltime-10))
    done
}

vm_life_cycle () {
    # prepare user data that need to be passed to VM
    echo "Preparing user data to be accessible inside VM (method: ${CLOUD_DATA_METHOD})..."
    setup_local_scratch
    userdata_copy_staged_in "$CLOUD_INFILES_LIST" "${CLOUD_USERDATA_DIR}"
    create_vm_data_disk ${CLOUD_DATA_METHOD} "${CLOUD_USERDATA_DIR}"
    echo_action "Done."

    # handle VM sysimage file and define params
    echo "Prepraing VM system image to run on WN hypervisor... "
    define_vm_params "${CLOUD_VM_DESCRIPTION}"
    prepare_vm_system_disk "${CLOUD_VM_SYSIMAGE}"
    [ -n "$guest_img_processing" ] && get_guest_os_info $VM_DISK_1
    define_vm_access_method
    echo_action "Done."

    # change password (if requested)
    [ -n "$RUNTIME_PASSWORD" ] && change_vm_access_password

    # get networking hardware configuration and start the VM
    lease_vm_mac
    start_vm ${HV_METHOD}
    wait_vm_network_ready

    # interactive access services should be started before we provides credentials to user
    echo "Waiting for interactive access service starts... "
    VM_ACCESS_SOCKET=$( get_interactive_access_socket $VM_NET_MAC $VM_LOCAL_PORT )
    if [ -n "$VM_ACCESS_SOCKET" ]; then
        echo_action "Done."
        print_interactive_access_credentials
        [ -n "${push_info}" ] && push_connect 
    else
        echo_action "Failed."
        destroy_vm
        exit_plus 1 "Cannot connect to ${VM_ACCESS_METHOD} port within $((VM_NET_CHECK_COUNT*VM_NET_CHECK_INTERVAL)) seconds. Destroying the machine."
    fi

    # sleep until VM is still running
    echo "Going to sleep for VM walltime ($VM_WALLTIME seconds)..."
    monitor_vm_is_running $VM_WALLTIME

    echo "Destroying the VM... "
    destroy_vm
    echo_action "Done."

    # handle output files before stage-out
    extract_vm_data_disk ${CLOUD_OUTFILES_LIST} ${CLOUD_DATA_METHOD} ${CLOUD_USERDATA_DIR}
}

#
# ARC GATEWAY EXECUTION 
# - RUNTIME RUNS FROM MAPPED USER ACCOUNT: NO ROOT ACEESS
#

if [ "x$1" = "x0" ]; then
    # go to working directory
    [ -n "$joboption_directory" ] && cd $joboption_directory
    # evaluate environment passed by ARC to worker node
    evaluate_arc_joboption_env
    # rainbow wrap-environment mode
    [ -n "$RAINBOW_JOB_WRAP" ] && wrap_job_environemt
     
    if [ -z "$RAINBOW_DISABLE_JOBOPTIONS_PROCESSING" ]; then
        # add VM params (grid job requested resources) to environment
        requested_resources_to_arc_env
        # save input and output files lists for RTE1 processing
        create_input_files_list "$CLOUD_INFILES_LIST"
        create_output_files_list "$CLOUD_OUTFILES_LIST"
    fi

    if [ -n "$NETHELPER_SOCKET" ]; then
        # source Rainbow Nethelper service client libs
        source $CLOUD_LIBEXEC_LOCATION/nethelper-client.sh
        # get nethelper auth parameters
        NETHELPER_AUTH=$( helper_auth_type )
        if [ "$NETHELPER_AUTH" = "cookie" ]; then
            cookie=$( helper_auth_getcookie $joboption_walltime )
            export NETHELPER_COOKIE=$cookie
            eval joboption_env_$env_idx="NETHELPER_COOKIE=$cookie"
            env_idx=$((env_idx+1))
        fi
    fi

    # push info about prepared job
    check_push_info_needed
    [ -n "$push_info" ] && push_prepared || true

elif [ "x$1" = "x1" ]; then

#
# WORKER NODE EXECUTION (START)
#

#
# HYPERVISOR ADAPTER SHOULD HANDLE:
#      VM_NAME
#      VM_CPUS (default 1)
#      VM_MEMORY_SIZE
#      VM_DISK_1 ... VM_DISK_N
#      VM_CD_DISK_1 ... VM_CD_DISK_N
#      VM_VVFAT_1 ... VM_VVFAT_N (optional for qemu)
#      VM_NET_TYPE (direct/bridge)
#      VM_NET_INT (physycal iface or bridge)      
#      VM_NET_MAC

    # temp within grid scratch 
    # ERROR: For some reason libguestfs failed to work with such temp
    # TMPDIR=$(mktemp -d --tmpdir=$PWD)
    # chmod 1777 $TMPDIR

    # redirect stdout/stderr to file
    [ -n "$RUNTIME_STDOUT" ] && exec 1>$RUNTIME_STDOUT
    [ -n "$RUNTIME_STDERR" ] && exec 2>$RUNTIME_STDERR
    # init cleanup script
    init_cleanup_script
    # load push info provider if needed
    check_push_info_needed

    # source Rainbow Nethelper service client libs
    source $CLOUD_LIBEXEC_LOCATION/nethelper-client.sh
    export NETHELPER_AUTH=$( helper_auth_type )

    # source disk images processing libs
    IMG_METHOD_LIB="$CLOUD_LIBEXEC_LOCATION/img_adapter_${IMG_METHOD:-essentials}.sh"
    if [ -f "$IMG_METHOD_LIB" ]; then
        source "$IMG_METHOD_LIB"
    else
        exit_plus 1 "Failed. Required data image processing libraries are missing. Check WN configuration."
    fi

    # source guest image processing libs (if installed)
    if [ -n "$GUEST_METHOD" ]; then
        GUEST_METHOD_LIB="$CLOUD_LIBEXEC_LOCATION/guest_adapter_${GUEST_METHOD}.sh"
        [ -f "$GUEST_METHOD_LIB" ] && source "$GUEST_METHOD_LIB"
        export guest_img_processing=1
    fi

    # source WN staging libs (if installed)
    if [ -n "$STAGING_METHOD" ]; then
        STAGING_METHOD_LIB="$CLOUD_LIBEXEC_LOCATION/staging_adapter_${STAGING_METHOD}.sh"
        [ -f "$STAGING_METHOD_LIB" ] && source "$STAGING_METHOD_LIB"
    fi

    # pilots handling (if requested) 
    # pilot processing should be done directly in the sourced libexec file
    if [ -n "$PILOT" ]; then
        PILOT_METHOD_LIB="$CLOUD_LIBEXEC_LOCATION/pilot_adapter_${PILOT}.sh"
        [ -f "$PILOT_METHOD_LIB" ] && source "$PILOT_METHOD_LIB"
    else
        vm_life_cycle
    fi

    # nethelper cookies cleanup
    if [ "$NETHELPER_AUTH" = "cookie" ]; then
        echo "helper_auth_delcookie \"${NETHELPER_COOKIE}\"" | add2_cleanup_script
    fi

    [ -n "$RUNTIME_STDOUT" ] && cp $RUNTIME_STDOUT ${RUNTIME_STDOUT}.rt1
    [ -n "$RUNTIME_STDERR" ] && cp $RUNTIME_STDERR ${RUNTIME_STDERR}.rt1

#
# WORKER NODE EXECUTION (FINISH)
#
elif [ "x$1" = "x2" ]; then
    check_push_info_needed
    [ -n "${push_info}" ] && push_finished
    cat ${RUNTIME_STDOUT}.rt1
    cat ${RUNTIME_STDERR}.rt1 >&2
    echo "Rainbow execution cycle has finished."
fi