#!/usr/bin/env python3
#===============================================================================
# Copyright 2012 NetApp, Inc. All Rights Reserved,
# contribution by Jorge Mora <mora@netapp.com>
#
# This program is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free Software
# Foundation; either version 2 of the License, or (at your option) any later
# version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
#===============================================================================
import os
import time
import errno
import fcntl
import struct
import traceback
import nfstest_config as c
from packet.nfs.nfs3_const import *
from packet.nfs.nfs4_const import *
from nfstest.test_util import TestUtil

# Module constants
__author__    = "Jorge Mora (%s)" % c.NFSTEST_AUTHOR_EMAIL
__copyright__ = "Copyright (C) 2012 NetApp, Inc."
__license__   = "GPL v2"
__version__   = "1.9"

USAGE = """%prog --server <server> [--client <client>] [options]

Delegation tests
================
Basic delegation tests verify that a correct delegation is granted when
opening a file for reading or writing. Also, another OPEN should not be
sent for the same file when the client is holding a delegation. Verify
that the stateid of all I/O operations should be the delegation stateid.
Reads from a different process on the same file should not cause the client
to send additional READ packets when the client is holding a read delegation.
Furthermore, a LOCK packet should not be sent to the server when the client
is holding a delegation.

Recall delegation tests verify the delegation is recalled when a conflicting
operation is sent to the server from a different client. Conflicting operations
are reading, writing, removing, renaming and changing the permissions on the
same file. Note that reading a file from a different client can only recall
a write delegation. Removing the delegated file from a different client
recalls the delegation and the server may or may not allow any more writes
from the client after the delegation has been returned. Renaming either the
delegated file (as source) or into the delegated file (as target) recalls
the delegation. In the case where the delegated file is the target of rename,
the existing target is removed before the rename occurs, therefore the server
may or may not allow nay more writes from the client after the delegation has
been removed just like in the case when removing the delegated file.

Also, verify that a read delegation is not recalled when a different client is
granted a read delegation. After a delegation is recalled, the client may send
an OPEN with CLAIM_DELEGATE_CUR before returning the delegation specially when
there is a open pending on the client. In addition, the stateid returned by the
new open should be the same as the original OPEN stateid. Also, a delegation
should not be granted when re-opening the file before returning the delegation.
The client may flush all written data before returning the WRITE delegation.
The LOCK should be sent as well before returning a delegation which has been
recalled. Finally, a delegation should not be granted on the second client who
cause the delegation recall on the first client.

Examples:
    Run the basic delegation tests (no client option):
        %prog --server 192.168.0.2 --export /exports

    Use short options instead:
        %prog -s 192.168.0.2 -e /exports

    Run both the basic and recall tests using positional arguments with
    nfsversion=3 for the second client:
        %prog -s 192.168.0.2 -e /exports --client 192.168.0.10:::3

    Use named arguments instead:
        %prog -s 192.168.0.2 -e /exports --client 192.168.0.10:nfsversion=3

Notes:
    The user id in the local host and the host specified by --client must
    have access to run commands as root using the 'sudo' command without
    the need for a password.

    The user id must be able to 'ssh' to remote host without the need for
    a password."""

# Test script ID
SCRIPT_ID = "DELEGATION"

# Test group flags
GROUP_BASIC    = (1 <<  0)  # Basic tests
GROUP_RECALL   = (1 <<  1)  # Recall tests
GROUP_RDELEG   = (1 <<  2)  # Read delegation tests
GROUP_WDELEG   = (1 <<  3)  # Write delegation tests
GROUP_IOREAD   = (1 <<  4)  # Tests with READ open
GROUP_IOWRTE   = (1 <<  5)  # Tests with WRITE open
GROUP_IORWRD   = (1 <<  6)  # Tests with RDWR open while reading
GROUP_IORWWR   = (1 <<  7)  # Tests with RDWR open while writing
GROUP_STAT     = (1 <<  8)  # Tests with file stat before open
GROUP_LOCK     = (1 <<  9)  # Tests with file lock after open
GROUP_CTREAD   = (1 << 10)  # Recall tests with conflicting READ
GROUP_CTWRTE   = (1 << 11)  # Recall tests with conflicting WRITE
GROUP_SETATTR  = (1 << 12)  # Recall tests by SETATTR
GROUP_REMOVE   = (1 << 13)  # Recall tests by removing the file
GROUP_RENAME   = (1 << 14)  # Recall tests by renaming the file
GROUP_PENDING  = (1 << 15)  # Recall tests having a pending open
GROUP_TARGET   = (1 << 16)  # Recall tests by renaming into the file

TESTNAMES_ALL = [
    ( "basic01",  GROUP_BASIC|GROUP_RDELEG|GROUP_IOREAD ),
    ( "basic02",  GROUP_BASIC|GROUP_WDELEG|GROUP_IOWRTE ),
    ( "basic03",  GROUP_BASIC|GROUP_RDELEG|GROUP_IOREAD|GROUP_STAT ),
    ( "basic04",  GROUP_BASIC|GROUP_WDELEG|GROUP_IOWRTE|GROUP_STAT ),
    ( "basic05",  GROUP_BASIC|GROUP_RDELEG|GROUP_IOREAD|GROUP_LOCK ),
    ( "basic06",  GROUP_BASIC|GROUP_WDELEG|GROUP_IOWRTE|GROUP_LOCK ),
    ( "basic07",  GROUP_BASIC|GROUP_WDELEG|GROUP_IORWRD ),
    ( "basic08",  GROUP_BASIC|GROUP_WDELEG|GROUP_IORWWR ),
    ( "basic09",  GROUP_BASIC|GROUP_WDELEG|GROUP_IORWRD|GROUP_STAT ),
    ( "basic10",  GROUP_BASIC|GROUP_WDELEG|GROUP_IORWWR|GROUP_STAT ),
    ( "basic11",  GROUP_BASIC|GROUP_WDELEG|GROUP_IORWRD|GROUP_LOCK ),
    ( "basic12",  GROUP_BASIC|GROUP_WDELEG|GROUP_IORWWR|GROUP_LOCK ),
    ( "recall01", GROUP_RECALL|GROUP_RDELEG|GROUP_IOREAD|GROUP_CTWRTE ),
    ( "recall02", GROUP_RECALL|GROUP_WDELEG|GROUP_IOWRTE|GROUP_CTWRTE ),
    ( "recall03", GROUP_RECALL|GROUP_RDELEG|GROUP_IOREAD|GROUP_CTWRTE ),
    ( "recall04", GROUP_RECALL|GROUP_WDELEG|GROUP_IOWRTE|GROUP_CTWRTE ),
    ( "recall05", GROUP_RECALL|GROUP_WDELEG|GROUP_IOWRTE|GROUP_CTREAD ),
    ( "recall06", GROUP_RECALL|GROUP_WDELEG|GROUP_IOWRTE|GROUP_CTREAD ),
    ( "recall07", GROUP_RECALL|GROUP_RDELEG|GROUP_IOREAD|GROUP_SETATTR ),
    ( "recall08", GROUP_RECALL|GROUP_WDELEG|GROUP_IOWRTE|GROUP_SETATTR ),
    ( "recall09", GROUP_RECALL|GROUP_RDELEG|GROUP_IOREAD|GROUP_SETATTR ),
    ( "recall10", GROUP_RECALL|GROUP_WDELEG|GROUP_IOWRTE|GROUP_SETATTR ),
    ( "recall11", GROUP_RECALL|GROUP_RDELEG|GROUP_IOREAD|GROUP_REMOVE ),
    ( "recall12", GROUP_RECALL|GROUP_WDELEG|GROUP_IOWRTE|GROUP_REMOVE ),
    ( "recall13", GROUP_RECALL|GROUP_RDELEG|GROUP_IOREAD|GROUP_REMOVE ),
    ( "recall14", GROUP_RECALL|GROUP_WDELEG|GROUP_IOWRTE|GROUP_REMOVE ),
    ( "recall15", GROUP_RECALL|GROUP_RDELEG|GROUP_IOREAD|GROUP_RENAME ),
    ( "recall16", GROUP_RECALL|GROUP_WDELEG|GROUP_IOWRTE|GROUP_RENAME ),
    ( "recall17", GROUP_RECALL|GROUP_RDELEG|GROUP_IOREAD|GROUP_RENAME ),
    ( "recall18", GROUP_RECALL|GROUP_WDELEG|GROUP_IOWRTE|GROUP_RENAME ),
    ( "recall19", GROUP_RECALL|GROUP_RDELEG|GROUP_IOREAD|GROUP_RENAME|GROUP_TARGET ),
    ( "recall20", GROUP_RECALL|GROUP_WDELEG|GROUP_IOWRTE|GROUP_RENAME|GROUP_TARGET ),
    ( "recall21", GROUP_RECALL|GROUP_RDELEG|GROUP_IOREAD|GROUP_RENAME|GROUP_TARGET ),
    ( "recall22", GROUP_RECALL|GROUP_WDELEG|GROUP_IOWRTE|GROUP_RENAME|GROUP_TARGET ),
    ( "recall23", GROUP_RECALL|GROUP_RDELEG|GROUP_IOREAD|GROUP_CTWRTE|GROUP_PENDING ),
    ( "recall24", GROUP_RECALL|GROUP_RDELEG|GROUP_IOREAD|GROUP_CTWRTE|GROUP_PENDING ),
    ( "recall25", GROUP_RECALL|GROUP_WDELEG|GROUP_IOWRTE|GROUP_CTWRTE|GROUP_PENDING ),
    ( "recall26", GROUP_RECALL|GROUP_WDELEG|GROUP_IOWRTE|GROUP_CTWRTE|GROUP_PENDING ),
    ( "recall27", GROUP_RECALL|GROUP_WDELEG|GROUP_IORWRD|GROUP_CTREAD ),
    ( "recall28", GROUP_RECALL|GROUP_WDELEG|GROUP_IORWWR|GROUP_CTREAD ),
    ( "recall29", GROUP_RECALL|GROUP_WDELEG|GROUP_IORWRD|GROUP_CTWRTE ),
    ( "recall30", GROUP_RECALL|GROUP_WDELEG|GROUP_IORWWR|GROUP_CTWRTE ),
    ( "recall31", GROUP_RECALL|GROUP_WDELEG|GROUP_IORWRD|GROUP_CTREAD ),
    ( "recall32", GROUP_RECALL|GROUP_WDELEG|GROUP_IORWWR|GROUP_CTREAD ),
    ( "recall33", GROUP_RECALL|GROUP_WDELEG|GROUP_IORWRD|GROUP_CTWRTE ),
    ( "recall34", GROUP_RECALL|GROUP_WDELEG|GROUP_IORWWR|GROUP_CTWRTE ),
    ( "recall35", GROUP_RECALL|GROUP_WDELEG|GROUP_IORWRD|GROUP_SETATTR ),
    ( "recall36", GROUP_RECALL|GROUP_WDELEG|GROUP_IORWWR|GROUP_SETATTR ),
    ( "recall37", GROUP_RECALL|GROUP_WDELEG|GROUP_IORWRD|GROUP_SETATTR ),
    ( "recall38", GROUP_RECALL|GROUP_WDELEG|GROUP_IORWWR|GROUP_SETATTR ),
    ( "recall39", GROUP_RECALL|GROUP_WDELEG|GROUP_IORWRD|GROUP_REMOVE ),
    ( "recall40", GROUP_RECALL|GROUP_WDELEG|GROUP_IORWWR|GROUP_REMOVE ),
    ( "recall41", GROUP_RECALL|GROUP_WDELEG|GROUP_IORWRD|GROUP_REMOVE ),
    ( "recall42", GROUP_RECALL|GROUP_WDELEG|GROUP_IORWWR|GROUP_REMOVE ),
    ( "recall43", GROUP_RECALL|GROUP_WDELEG|GROUP_IORWRD|GROUP_RENAME ),
    ( "recall44", GROUP_RECALL|GROUP_WDELEG|GROUP_IORWWR|GROUP_RENAME ),
    ( "recall45", GROUP_RECALL|GROUP_WDELEG|GROUP_IORWRD|GROUP_RENAME ),
    ( "recall46", GROUP_RECALL|GROUP_WDELEG|GROUP_IORWWR|GROUP_RENAME ),
    ( "recall47", GROUP_RECALL|GROUP_WDELEG|GROUP_IORWRD|GROUP_RENAME|GROUP_TARGET ),
    ( "recall48", GROUP_RECALL|GROUP_WDELEG|GROUP_IORWWR|GROUP_RENAME|GROUP_TARGET ),
    ( "recall49", GROUP_RECALL|GROUP_WDELEG|GROUP_IORWRD|GROUP_RENAME|GROUP_TARGET ),
    ( "recall50", GROUP_RECALL|GROUP_WDELEG|GROUP_IORWWR|GROUP_RENAME|GROUP_TARGET ),
    ( "recall51", GROUP_RECALL|GROUP_WDELEG|GROUP_IORWRD|GROUP_CTWRTE|GROUP_PENDING ),
    ( "recall52", GROUP_RECALL|GROUP_WDELEG|GROUP_IORWRD|GROUP_CTWRTE|GROUP_PENDING ),
    ( "recall53", GROUP_RECALL|GROUP_WDELEG|GROUP_IORWWR|GROUP_CTWRTE|GROUP_PENDING ),
    ( "recall54", GROUP_RECALL|GROUP_WDELEG|GROUP_IORWWR|GROUP_CTWRTE|GROUP_PENDING ),
]

TESTNAMES_DICT = dict(TESTNAMES_ALL)

def group_test(tname, group):
    """Return True if test belongs to the given group"""
    testgroup = TESTNAMES_DICT.get(tname)
    if testgroup is not None and (testgroup & group) == group:
        return True
    return False

def group_list(group):
    """Return a list of tests belonging to the given group"""
    return [x[0] for x in TESTNAMES_ALL if group_test(x[0], group)]

TESTNAMES_BASIC  = group_list(GROUP_BASIC)
TESTNAMES_RECALL = group_list(GROUP_RECALL)

# Include the test groups in the list of test names
# so they are displayed in the help
GTESTS = ["recall", "setattr", "remove", "rename", "pending"]
TESTNAMES = ["basic", "stat", "lock"] + TESTNAMES_BASIC + GTESTS + \
             TESTNAMES_RECALL + ["read_deleg", "write_deleg"]

TESTGROUPS = {
    "basic": {
         "tests": TESTNAMES_BASIC,
         "desc": "Run all basic delegation tests: ",
    },
    "stat": {
         "tests": group_list(GROUP_STAT),
         "desc": "Run all basic delegation tests with file stat: ",
    },
    "lock": {
         "tests": group_list(GROUP_LOCK),
         "desc": "Run all basic delegation tests with file lock: ",
    },
    "recall": {
         "tests": TESTNAMES_RECALL,
         "desc": "Run all recall delegation tests: ",
    },
    "setattr": {
         "tests": group_list(GROUP_SETATTR),
         "desc": "Run all tests using SETATTR to recall the delegation: ",
    },
    "remove": {
         "tests": group_list(GROUP_REMOVE),
         "desc": "Run all tests recalling the delegation by removing the delegated file: ",
    },
    "rename": {
         "tests": group_list(GROUP_RENAME),
         "desc": "Run all tests recalling the delegation by renaming the delegated file: ",
    },
    "pending": {
         "tests": group_list(GROUP_PENDING),
         "desc": "Run all recall delegation tests having a pending open: ",
    },
    "read_deleg": {
         "tests": group_list(GROUP_RDELEG),
         "desc": "Run all read delegation tests: ",
    },
    "write_deleg": {
         "tests": group_list(GROUP_WDELEG),
         "desc": "Run all write delegation tests: ",
    },
}

# Dictionary having the number of clients required by each test
TEST_CLIENT_DICT = {x:1 for x in TESTNAMES_RECALL}

PATTERN = b'FF00'

OPEN_STID  = 0
LOCK_STID  = 1
DELEG_STID = 2
stid_map = {
    OPEN_STID  : "OPEN",
    LOCK_STID  : "LOCK",
    DELEG_STID : "DELEG",
}

OPEN_READ  = 0
OPEN_WRITE = 1
OPEN_RDWR  = 2

open_flags = {
    OPEN_READ  : os.O_RDONLY,
    OPEN_WRITE : os.O_WRONLY|os.O_CREAT,
    OPEN_RDWR  : os.O_RDWR|os.O_CREAT,
}

open_str = {
    OPEN_READ  : "READ",
    OPEN_WRITE : "WRITE",
    OPEN_RDWR  : "RDWR",
}

deleg_map = {
    OPEN_READ  : OPEN_DELEGATE_READ,
    OPEN_WRITE : OPEN_DELEGATE_WRITE,
    OPEN_RDWR  : OPEN_DELEGATE_WRITE,
}

deleg_str = {
    OPEN_DELEGATE_READ  : "READ",
    OPEN_DELEGATE_WRITE : "WRITE",
}

def file_lock(fd, open_type, absfile, lock_offset=0, lock_len=0):
    """Lock file given by the file descriptor"""
    lock_type = fcntl.F_RDLCK if open_type == OPEN_READ else fcntl.F_WRLCK
    lockdata = struct.pack('hhllhh', lock_type, 0, lock_offset, lock_len, 0, 0)
    return fcntl.fcntl(fd, fcntl.F_SETLK, lockdata)

class BaseName(Exception):
    """Exception used to stop recall tests when --basename option is set"""
    pass

class DelegTest(TestUtil):
    """DelegTest object

       DelegTest() -> New test object

       Usage:
           x = DelegTest(testnames=['basic', 'basic_lock', ...])

           # Run all the tests
           x.run_tests(deleg=deleg_mode)
           x.exit()
    """
    def __init__(self, **kwargs):
        """Constructor

           Initialize object's private data.
        """
        TestUtil.__init__(self, **kwargs)
        self.test_opgroup.version = "%prog " + __version__

        # Options specific for this test script
        hhelp = "Remote NFS client and options used for recall delegation tests. " \
                "Clients are separated by a ',' and each client definition is " \
                "a list of arguments separated by a ':' given in the following " \
                "order if positional arguments is used (see examples): " \
                "clientname:server:export:nfsversion:port:proto:sec:mtpoint " \
                "[default: '%default']"
        self.test_opgroup.add_option("--client", default='nfsversion=3:proto=tcp:port=2049', help=hhelp)
        hhelp = "Comma separated list of valid NFS versions to use in the " \
                "--client option. An NFS version from this list, which is " \
                "different than that given by --nfsversion, is selected and " \
                "included in the --client option [default: %default]"
        self.test_opgroup.add_option("--client-nfsvers", default="4.0,4.1", help=hhelp)
        hhelp = "Starting offset for lock [default: %default]"
        self.test_opgroup.add_option("--lock-offset", type="int", default=0, help=hhelp)
        hhelp = "Starting offset for lock on pending open [default: %default]"
        self.test_opgroup.add_option("--lock-poffset", type="int", default=8192, help=hhelp)
        hhelp = "Number of bytes to lock [default: %default]"
        self.test_opgroup.add_option("--lock-len", type="int", default=4096, help=hhelp)
        hhelp = "Truncate file when writing from the second file for the recall tests"
        self.test_opgroup.add_option("--truncate", action="store_true", default=False, help=hhelp)
        hhelp = "Seconds to delay after setup so all opens are released [default: %default]"
        self.test_opgroup.add_option("--setup-delay", type="float", default=4.0, help=hhelp)
        self.scan_options()

        if len(self.basename) > 0:
            self.setup_delay = 0.0

        self.rfindex = 1
        self.nrfiles = 1
        for tname in self.testlist:
            if group_test(tname, GROUP_RDELEG) or \
               group_test(tname, GROUP_STAT)   or \
               group_test(tname, GROUP_REMOVE) or \
               group_test(tname, GROUP_RENAME) or \
               group_test(tname, GROUP_IORWRD) or \
               group_test(tname, GROUP_CTREAD):
                self.nrfiles += 1

            if group_test(tname, GROUP_TARGET):
                self.nrfiles += 1

        # Disable createtraces option
        self.createtraces = False

        # Local rexec object
        self.lexecobj = None

        # Find how many remote Rexec objects should be started
        nclients = 0
        for tname in self.testlist:
            nclients = max(nclients, TEST_CLIENT_DICT.get(tname, 0))

        # Process the --client option
        client_list = self.process_client_option(count=nclients, remote=None)
        if self.client_nfsvers is not None:
            nfsvers_list = self.str_list(self.client_nfsvers)
            for client_args in client_list:
                if self.proto[-1] == "6" and len(client_args.get("proto")) and client_args["proto"][-1] != 6:
                    client_args["proto"] += "6"
                for nfsver in nfsvers_list:
                    if nfsver != self.nfsversion:
                        client_args["nfsversion"] = nfsver
                        break
                else:
                    self.opts.error("At least one NFS version in --client-nfsvers '%s' " \
                                    "must be different then --nfsversion %s" % \
                                    (self.client_nfsvers, self.nfsversion))

        # Remove all client specs which are not valid -- when mount is 0 that
        # means it is the same client as the main client with the same mount
        # options.
        index = 0
        while index < len(client_list):
            if client_list[index].get("mount", 0) == 0:
                client_list.pop(index)
                continue
            index += 1

        self.verify_client_option(TEST_CLIENT_DICT)

        # Start remote procedure server(s) remotely
        try:
            self.clientobj = None
            for client_args in client_list:
                client_name = client_args.pop("client", "")
                self.create_host(client_name, **client_args)
                self.create_rexec(client_name)
        except:
            self.test(False, traceback.format_exc())

        # Verify the lock ranges do not overlap
        end1 = self.lock_offset  + self.lock_len - 1
        end2 = self.lock_poffset + self.lock_len - 1
        if end1 >= self.lock_poffset and end2 >= self.lock_offset:
            # Ranges overlap
            self.opts.error("Lock ranges overlap: (lock-offset, lock-len) and (lock-poffset, lock-len)")

    def setup(self):
        """Setup test environment"""
        # Call base object's setup method
        super(DelegTest, self).setup(nfiles=self.nrfiles)
        # Delay so all opens are released and delegations could be granted
        time.sleep(self.setup_delay)

    def lock_file(self, fd, open_type, absfile, lock_offset=0, lock_len=0):
        """Lock file given by the file descriptor.

           fd:
               Opened file descriptor of file
           open_type:
               Open type
           absfile:
               File path to display as debug info
           lock_offset:
               Lock offset [default: 0]
           lock_len:
               Lock length [default: 0]
        """
        lock_str = "F_RDLCK" if open_type == OPEN_READ else "F_WRLCK"
        try:
            fmsg = ""
            self.dprint('DBG3', "Lock %s (F_SETLK, %s) start=%d len=%d" % (absfile, lock_str, lock_offset, lock_len))
            file_lock(fd, open_type, absfile, lock_offset, lock_len)
        except OSError as exerr:
            fmsg = ", failed with %s" % exerr
        dmsg = "Lock file with %s" % lock_str
        self.test(len(fmsg) == 0, "%s should succeed" % dmsg, failmsg=fmsg)

    def open_file(self, absfile, open_type, io_type=None, lock=False, lexec=False, msg=''):
        """Open file, lock it and do some I/O on the file.
           Return the file descriptor of the opened file.

           absfile:
               File name to open
           open_type:
               Open type
           io_type:
               I/O type
           lock:
               Get a lock on the file if true [default: False]
           lexec:
               Use different process to open file [default: False]
           msg:
               Message to append on debug message [default: '']
        """
        pidstr = " from a different process" if lexec else ""
        msg = msg if len(msg) == 0 else " %s" % msg
        mode_str = open_str[open_type]
        if io_type is None:
            io_type = OPEN_READ if open_type == OPEN_READ else OPEN_WRITE
        io_str = open_str[io_type].capitalize()

        try:
            fmsg = ""
            dmsg = "Open file for %s%s%s" % (mode_str, pidstr, msg)
            self.dprint('DBG2', "%s [%s]" % (dmsg, absfile))
            if lexec:
                fd = self.lexecobj.run(os.open, absfile, open_flags[open_type])
            else:
                fd = os.open(absfile, open_flags[open_type])
        except OSError as exerr:
            fmsg = ", failed with %s" % exerr
        self.test(len(fmsg) == 0, "%s should succeed" % dmsg, failmsg=fmsg)

        if lock:
            self.lock_file(fd, io_type, absfile, self.lock_offset, self.lock_len)

        try:
            fmsg = ""
            dmsg = "%s file%s%s" % (io_str, pidstr, msg)
            self.dprint("DBG3", "%s [%s]" % (dmsg, absfile))
            # Read/Write file
            if io_type == OPEN_READ:
                if lexec:
                    self.lexecobj.run(os.read, fd, self.rsize)
                else:
                    os.read(fd, self.rsize)
            else:
                data = self.data_pattern(0, self.wsize, PATTERN)
                if lexec:
                    self.lexecobj.run(os.write, fd, data)
                else:
                    os.write(fd, data)
        except OSError as exerr:
            fmsg = ", failed with %s" % exerr
        self.test(len(fmsg) == 0, "%s should succeed" % dmsg, failmsg=fmsg)

        self.delay_io()
        return fd

    def get_deleg_remote(self):
        """Get a read delegation on the remote client."""
        fdko = None
        absfile = self.clientobj.abspath(self.filename)
        if self.clientobj and self.clientobj.nfs_version < 4:
            # There are no delegations in NFSv3 so there is no need
            # to open a file so the open owner sticks around
            self.dprint("DBG2", "Open file on the remote client [%s]" % absfile)
        else:
            # Open file so open owner sticks around so a delegation
            # is granted when opening the file under test
            fdko = self.rexecobj.run(os.open, self.clientobj.abspath(self.files[0]), os.O_RDONLY)
            self.dprint("DBG2", "Get a read delegation on the remote client [%s]" % absfile)
        # Open the file under test
        fdrd = self.rexecobj.run(os.open, absfile, os.O_RDONLY)
        self.dprint("DBG3", "Read %s on the remote client" % absfile)
        data = self.rexecobj.run(os.read, fdrd, 1024)
        self.dprint("DBG4", "Close %s on the remote client" % absfile)
        self.rexecobj.run(os.close, fdrd)
        if fdko is not None:
            self.rexecobj.run(os.close, fdko)

    def setup_test(self, io_type, mount=False, nfiles=0, lexec=False):
        """Setup test by mounting server and hold open a file so that the open
           owner sticks around so a delegation is granted on next open using
           the same open owner -- this is done to avoid a bug on the client
           where open owner is reaped at close
        """
        self.umount()
        if mount and self.clientobj is not None:
            # Unmount server on remote client
            self.clientobj.umount()

        if lexec and self.lexecobj is None:
            # Start local rexec connection just once
            rexecobj_save = self.rexecobj
            self.lexecobj = self.create_rexec()
            self.rexecobj = rexecobj_save
            self.lexecobj.rimport("fcntl")
            self.lexecobj.rcode(file_lock)

        self.trace_start()
        self.mount()
        if mount and self.clientobj is not None:
            # Mount server on remote client
            self.clientobj.mount()

        if io_type == OPEN_READ or nfiles > 0:
            # Use existing file
            self.filename = self.files[self.rfindex]
            self.absfile = self.abspath(self.filename)
            self.rfindex += 1
        else:
            # Create new file
            self.get_filename()

        # Hold a file open so that the open owner sticks around
        # (bug on the client where OO's are reaped at close)
        self.dprint('DBG4', "Open %s so open owner sticks around" % self.abspath(self.files[0]))
        self.fdko = open(self.abspath(self.files[0]), 'r')

    def verify_io_requests(self, iomode, deleg_stid, filehandles, src_ipaddr=None, maxindex=None):
        """Verify I/O is sent to the correct server."""
        nio = 0
        dsindex = 0
        for fh in filehandles:
            if self.dslist:
                # The address is one of the DS's connection
                ds = self.dslist[dsindex]
            else:
                # The address is the mounted server
                ds = [{"ipaddr": self.server_ipaddr, "port": self.port}]
            for item in ds:
                save_index = self.pktt.get_index()
                nio += self.verify_io(iomode, deleg_stid, item["ipaddr"], item["port"], filehandle=fh, src_ipaddr=src_ipaddr, maxindex=maxindex, pattern=PATTERN)
                self.pktt.rewind(save_index)
            dsindex += 1
        return nio

    def verify_open(self, fh, stat=False):
        """Verify OPEN call"""
        self.test(self.opencall, "OPEN should be sent")
        if self.opencall is None:
            return
        elif stat and self.nfs_version > 4.0:
            expr = self.opencall.NFSop.claim.claim == CLAIM_FH
            self.test(expr, "OPEN should be sent with CLAIM_FH")
            expr = self.opencall.NFSop.fh == fh
            self.test(expr, "OPEN should be sent with the filehandle of the file to be opened")
        else:
            expr = self.opencall.NFSop.claim.claim == CLAIM_NULL
            self.test(expr, "OPEN should be sent with CLAIM_NULL")
            if expr:
                expr = self.opencall.NFSop.claim.name == self.filename
                self.test(expr, "OPEN should be sent with the name of the file to be opened")
            expr = self.opencall.NFSop.fh != fh
            self.test(expr, "OPEN should be sent with the filehandle of the directory")

    def find_ios(self, op_type, filehandle, ipaddr, port):
        """Return a list of all I/O packets"""
        ret = {}
        # Matched all packets sent to the server given by ipaddr and port
        src = "IP.src == '%s' and " % self.client_ipaddr
        dst = self.pktt.ip_tcp_dst_expr(ipaddr, port)
        fh  = " and NFS.fh == b'%s' and " % self.pktt.escape(filehandle)
        nfsver = self.match_nfs_version(self.nfs_version, False)
        matchstr = src + dst + nfsver + fh + "NFS.argop == %d" % op_type

        save_index = self.pktt.get_index()
        self.pktt.clear_xid_list()
        try:
            # Matched all I/O packets and their replies
            while self.pktt.match(matchstr, reply=True):
                pkt = self.pktt.pkt
                xid = pkt.rpc.xid
                if pkt.rpc.type == 0:
                    # Save I/O call info
                    nfsop = pkt.NFSop
                    info = {
                        "stateid": nfsop.stateid,
                        "count":   nfsop.count,
                        "nfsidx":  pkt.NFSidx,
                        "callidx": pkt.record.index,
                    }
                    if ret.get(xid) is None:
                        ret[xid] = info
                    else:
                        ret[xid].update(info)
                else:
                    # Save I/O reply status
                    idx = ret[xid].get("nfsidx")
                    if idx is not None and len(pkt.nfs.array) > idx:
                        nfsop = pkt.nfs.array[idx]
                        ret[xid]["status"] = nfsop.status
        except:
            self.test(False, traceback.format_exc())
        finally:
            self.pktt.rewind(save_index)
        # Return the list of I/O packets having status values
        return [item for item in ret.values() if item.get("status") is not None]

    def find_io_counts(self, io_list, stateid, status):
        """Return the number of matched stateid and matched status packets"""
        stid = 0  # Number of I/O packets matching the stateid
        stat = 0  # Number of I/O packets matching the status
        okct = 0  # Number of I/O packets with status = NFS4_OK
        for item in io_list:
            istatus = item.get('status')
            if istatus is None:
                continue
            if item.get('stateid') == stateid:
                stid += 1
            if istatus == status:
                stat += 1
            if istatus == NFS4_OK:
                okct += 1
        return (stid, stat, okct)

    def verify_io_per_server(self, io_list, op_type, stid_type, stateid, status, ds=False, delegret=False):
        """Verify I/O packets for a given server"""
        dr_str = " after returning the delegation" if delegret else ""
        io_str = "READ" if op_type == OP_READ else "WRITE"
        st_str = stid_map.get(stid_type)
        sv_str = "server"
        if self.layout and self.dslist:
            sv_str = "DS" if ds else "MDS"

        if io_list:
            nlen = len(io_list)
            (stid, stat, okct) = self.find_io_counts(io_list, stateid, status[0])
            self.test(stid == nlen, "%ss should be sent to the %s with the %s stateid%s" % (io_str, sv_str, st_str, dr_str))
            if delegret and okct == nlen and stat == 0:
                # The RFC allows servers to process I/O operations successfully
                # when the file has been removed. In this case all I/O operations
                # have succeeded but an error was expected for those servers
                # failing I/O operations when the file is removed.
                self.test(True, "%ss may return NFS4_OK from the %s%s" % (io_str, sv_str, dr_str))
            else:
                # Check if all I/O operations returned one of the expected
                # status codes
                idx = 0
                if stat != nlen:
                    for i in range(1, len(status)):
                        (stid, stat, okct) = self.find_io_counts(io_list, stateid, status[i])
                        if stat == nlen:
                            idx = i
                            break
                self.test(stat == nlen, "%ss should return %s from the %s%s" % (io_str, nfsstat4.get(status[idx], status[idx]), sv_str, dr_str))

    def verify_io_packets(self, op_type, open_fh, stid_type, stateid, status=[NFS4_OK], ds_status=[NFS4_OK], delegret=False):
        """Verify I/O packets"""
        io_list = []
        if self.layout and self.dslist:
            # Get I/O packets sent to the DS
            dsindex = 0
            for fh in self.layout['filehandles']:
                for item in self.dslist[dsindex]:
                    if item is not None:
                        io_list += self.find_ios(op_type, fh, item['ipaddr'], item['port'])
                dsindex += 1
        # Verify I/O packets sent to the DS
        self.verify_io_per_server(io_list, op_type, stid_type, stateid, ds_status, ds=True, delegret=delegret)

        # Verify I/O packets sent to the server (or MDS if pNFS is available)
        io_list = self.find_ios(op_type, open_fh, self.server_ipaddr, self.port)
        self.verify_io_per_server(io_list, op_type, stid_type, stateid, status, delegret=delegret)

    def verify_lock(self, io_type, mode_str, open_stid, offset, start_index, max_index, msg="", locker=None, lock_stid=None):
        """Verify correct lock is sent to the server"""
        lock_stateid = None
        self.pktt.rewind(start_index)
        # Find LOCK call and reply using the lock offset
        mstr = "NFS.offset == %d" % offset
        (lockcall, lockreply) = self.find_nfs_op(OP_LOCK, match=mstr, status=None, src_ipaddr=self.client_ipaddr, maxindex=max_index)
        self.test(lockcall, "LOCK should be sent before returning the %s delegation%s" % (mode_str, msg))
        if lockcall:
            # Verify lock info sent to the server
            ltype = READ_LT if io_type == OPEN_READ else WRITE_LT
            self.test(lockcall.NFSop.locktype == ltype, "LOCK should be sent with correct lock type")
            self.test(lockcall.NFSop.length == self.lock_len, "LOCK should be sent with correct lock range")
            pktlocker = lockcall.NFSop.locker
            if pktlocker.new_lock_owner:
                lowner = pktlocker.open_owner
            else:
                lowner = pktlocker.lock_owner
            self.test(lowner.stateid == open_stid, "LOCK should be sent with correct OPEN stateid")
            if locker is not None:
                # Verify lock has a different lock owner than the one given
                if locker.new_lock_owner and pktlocker.new_lock_owner:
                    # Both locks are sent with new lock owners
                    expr = locker.open_owner.lock_owner != pktlocker.open_owner.lock_owner.owner
                elif not locker.new_lock_owner and not pktlocker.new_lock_owner:
                    # Both locks are sent with existing lock owners
                    expr = locker.lock_owner.stateid != pktlocker.lock_owner.lock_owner.stateid
                else:
                    # One lock is sent with a new lock owner and the other
                    # with and existing lock owner
                    expr = True
                self.test(expr, "LOCK should be sent with a different open owner")
        if lockreply:
            # Verify lock reply
            lstatus = lockreply.nfs.status
            fmsg = ", failed with %s" % nfsstat4.get(lstatus, lstatus)
            self.test(lstatus == NFS4_OK, "LOCK should return NFS4_OK", failmsg=fmsg)
            if lockreply.nfs.status == NFS4_OK:
                lock_stateid = lockreply.NFSop.stateid.other
                if lock_stid is not None:
                    # Verify lock stateid is different than the one given
                    expr = lock_stid != lock_stateid
                    self.test(expr, "LOCK should return a different lock stateid")
        return lock_stateid

    def basic_deleg_test(self, open_type, io_type=None, lock=False, stat=False, nfiles=0):
        """Basic delegation tests"""
        try:
            fds = []
            extra_str = ""
            self.fdko = None
            deleg_type = deleg_map[open_type]
            mode_str = deleg_str[deleg_type]
            if io_type is None:
                io_type = OPEN_READ if open_type == OPEN_READ else OPEN_WRITE
            if open_type == OPEN_RDWR:
                io_str = "reading" if io_type == OPEN_READ else "writing"
                extra_str = " using RDWR open while %s" % io_str
            if lock:
                extra_str += " with file lock"
            elif stat:
                extra_str += " with file stat"
            self.test_group("Basic %s delegation test%s" % (mode_str, extra_str))
            self.setup_test(io_type, nfiles=nfiles, lexec=True)

            if stat:
                try:
                    fmsg = ""
                    dmsg = "Stat file to cache file metadata"
                    self.dprint('DBG3', "%s [%s]" % (dmsg, self.absfile))
                    fstat = os.stat(self.absfile)
                except OSError as exerr:
                    fmsg = ", failed with %s" % exerr
                self.test(len(fmsg) == 0, "%s should succeed" % dmsg, failmsg=fmsg)

            # Open file, should get a DELEGATION
            fds.append(self.open_file(self.absfile, open_type, io_type, lock=lock))

            # Open same file on same process for reading
            fds.append(self.open_file(self.absfile, OPEN_READ, msg="on same process"))

            if open_type == OPEN_WRITE:
                # Open same file on same process for writing
                fds.append(self.open_file(self.absfile, OPEN_WRITE, msg="on same process"))

            # Access file from a different process
            fd = self.open_file(self.absfile, OPEN_READ, lexec=True)
            self.lexecobj.run(os.close, fd)

            if open_type == OPEN_WRITE:
                # Open same file on different process for writing
                fd = self.open_file(self.absfile, OPEN_WRITE, lexec=True)
                self.lexecobj.run(os.close, fd)
        except Exception:
            self.test(False, traceback.format_exc())
            return
        finally:
            if self.fdko:
                # Close open owner file
                self.fdko.close()
            # Close open files
            for fd in fds:
                os.close(fd)
            self.umount()
            self.trace_stop()

        try:
            self.trace_open()
            self.set_pktlist()

            filehandle = None
            if stat:
                # Find the LOOKUP for the file under test
                while True:
                    (lookupcall, lookupreply) = self.find_nfs_op(OP_LOOKUP, src_ipaddr=self.client_ipaddr)
                    if lookupcall is None or lookupcall.NFSop.name == self.filename:
                        # Found LOOKUP for the filename or the end of the
                        # trace file has been reached
                        break
                self.test(lookupcall, "LOOKUP operation should be sent")
                if lookupreply:
                    # GETFH should be the operation following the LOOKUP
                    getfh_obj = self.getop(lookupreply, OP_GETFH)
                    if getfh_obj:
                        # Get the file handle for the file under test
                        filehandle = getfh_obj.fh
                    else:
                        # Could not find GETFH
                        self.test(False, "Could not find GETFH operation in the LOOKUP compound")

            (fh, op_stid, deleg_stid) = self.find_open(filename=self.filename, claimfh=filehandle, deleg_type=deleg_type, anyclaim=True)
            self.verify_open(fh, stat)
            self.test(deleg_stid != None, "%s delegation should be granted" % mode_str)
            save_index = self.pktt.get_index()
            if deleg_stid is None:
                # Delegation was not granted
                return

            filehandles = [fh]
            self.find_layoutget(fh)
            (devcall, devreply, dslist) = self.find_getdeviceinfo()
            self.pktt.rewind(save_index)
            if self.layout:
                filehandles = self.layout['filehandles']

            # Find any other OPENs for the same file
            olist = self.find_open(filename=self.filename)
            self.test(olist[0] is None, "OPEN should not be sent for the same file")

            if lock:
                # Rewind trace file
                self.pktt.rewind()
                (lockcall, lockreply) = self.find_nfs_op(OP_LOCK, src_ipaddr=self.client_ipaddr)
                self.test(lockcall is None, "LOCK should not be sent to the server")

            # Verify I/O packets
            op_type = OP_READ if open_type == OPEN_READ else OP_WRITE
            self.verify_io_packets(op_type, fh, DELEG_STID, deleg_stid)

            # Rewind trace file to saved packet index
            if deleg_type == OPEN_DELEGATE_READ:
                self.pktt.rewind(save_index)
                nio = self.verify_io_requests(deleg_type, deleg_stid, filehandles, src_ipaddr=self.client_ipaddr)
                if nio > 0:
                    unique_io_list = sorted(set(self.test_offsets))
                    expr = len(self.test_offsets) == len(unique_io_list)
                    self.test(expr, "%s should not be sent when reading delegated file from a different process" % mode_str)

            # Find CLOSE request and reply
            self.verify_close(fh, op_stid, pindex=save_index)

            if self.pktcall:
                # Find DELEGRETURN request and reply
                self.pktt.rewind(self.pktcall.record.index)
                match_str = "NFS.fh == b'%s'" % self.pktt.escape(fh)
                (delegreturncall, delegreturnreply) = self.find_nfs_op(OP_DELEGRETURN, src_ipaddr=self.client_ipaddr, match=match_str)
                self.test(delegreturncall, "DELEGRETURN should be sent after the close")
                if delegreturncall:
                    expr = delegreturncall.NFSop.stateid.other == deleg_stid
                    self.test(expr, "DELEGRETURN should be sent with the delegation stateid")
        except Exception:
            self.test(False, traceback.format_exc())
        finally:
            self.pktt.close()

    def get_conflict_pkts(self, conflict_op, conflict_match):
        """Get conflicting operation packets"""
        pkt_call = None
        pktcall  = None
        pktreply = None
        while True:
            pkt_call = self.pktt.match(conflict_match)
            if pkt_call is None:
                break
            elif pktcall is None:
                # Get the first OPEN call for testing CB_RECALL
                pktcall = pkt_call

            # Make sure conflicting operation reply status from the
            # second client is not NFS4ERR_DELAY
            xid = pkt_call.rpc.xid
            matchstr = "RPC.xid == %d and NFS.status in (%d, %d) and NFS.resop == %d" % (xid, NFS4_OK, NFS4ERR_DELAY, conflict_op)
            pktreply = self.pktt.match(matchstr)
            if pktreply is None or pktreply.nfs.status == NFS4_OK:
                break
        return pktcall, pktreply

    def recall_deleg_test(self, open_type, io_type=None, conflict_type=OP_WRITE, lock=False, nfiles=0, target=False, claim_cur=None):
        """Delegation recall tests"""
        if self.clientobj is None:
            return
        try:
            fd = None
            fdsec = None
            self.fdko = None
            extra_str = ""
            deleg_type = deleg_map[open_type]
            mode_str = deleg_str[deleg_type]
            nfs_version = self.clientobj.nfs_version
            sipaddr = self.clientobj.ipaddr
            sproto = self.clientobj.proto
            sport = self.clientobj.port
            if io_type is None:
                io_type = OPEN_READ if open_type == OPEN_READ else OPEN_WRITE
            io_str = open_str[io_type]
            op_type = OP_READ if io_type == OPEN_READ else OP_WRITE
            if conflict_type == OP_SETATTR:
                conflict_str = "SETATTR (chmod)"
                conflict_op = OP_SETATTR if nfs_version > 3 else NFSPROC3_SETATTR
            elif conflict_type == OP_REMOVE:
                conflict_str = "REMOVE"
                conflict_op = OP_REMOVE if nfs_version > 3 else NFSPROC3_REMOVE
            elif conflict_type == OP_RENAME:
                conflict_str = "RENAME"
                conflict_op = OP_RENAME if nfs_version > 3 else NFSPROC3_RENAME
                conflict_str += " (DST)" if target else " (SRC)"
            elif nfs_version < 4:
                if conflict_type == OP_READ:
                    conflict_str = "READ"
                    conflict_op = NFSPROC3_READ
                    # Use an existing file because reading an empty file
                    # in NFSv3 will not send the READ procedure
                    nfiles = 1
                else:
                    conflict_str = "WRITE"
                    conflict_op = NFSPROC3_WRITE
            else:
                nfiles = 1 if conflict_type == OP_READ else 0
                ctype = "READ" if conflict_type == OP_READ else "WRITE"
                conflict_str = "OPEN (%s)" % ctype
                conflict_op = OP_OPEN

            if open_type == OPEN_RDWR:
                iostr = "reading" if io_type == OPEN_READ else "writing"
                extra_str = " using RDWR open while %s" % iostr

            lock_str = " with file lock" if lock else ""

            claim_str = ""
            if claim_cur == os.O_RDONLY:
                claim_io = "READ"
                claim_str = ", having a pending READ open"
            elif claim_cur == os.O_WRONLY:
                claim_io = "WRITE"
                claim_str = ", having a pending WRITE open"

            # Flag to read same file from another client to test the
            # delegation should not be recalled
            other_read_deleg = deleg_type == OPEN_DELEGATE_READ and claim_cur is None

            extra_str += "%s%s" % (lock_str, claim_str)
            self.test_group("Recall %s delegation with %s%s" % (mode_str, conflict_str, extra_str))
            self.setup_test(io_type, mount=True, nfiles=nfiles, lexec=(claim_cur is not None))

            # Open file, should get a DELEGATION
            try:
                fmsg = ""
                dmsg = "Open file for %s" % open_str[open_type]
                self.dprint('DBG2', "%s [%s]" % (dmsg, self.absfile))
                fd = os.open(self.absfile, open_flags[open_type])
            except OSError as exerr:
                fmsg = ", failed with %s" % exerr
            self.test(len(fmsg) == 0, "%s should succeed" % dmsg, failmsg=fmsg)

            if lock:
                self.lock_file(fd, io_type, self.absfile, self.lock_offset, self.lock_len)

            iosize = int(self.PAGESIZE/2)

            cio_type = None
            if claim_cur is not None:
                try:
                    fmsg = ""
                    dmsg = "Open file for %s in a different process" % claim_io
                    self.dprint('DBG2', "%s [%s]" % (dmsg, self.absfile))
                    fdsec = self.lexecobj.run(os.open, self.absfile, claim_cur)
                except OSError as exerr:
                    fmsg = ", failed with %s" % exerr
                self.test(len(fmsg) == 0, "%s should succeed" % dmsg, failmsg=fmsg)

                try:
                    fmsg = ""
                    dmsg = "Lock file in a different process"
                    cio_type = OPEN_READ if claim_cur == os.O_RDONLY else OPEN_WRITE
                    cio_lstr = "F_RDLCK" if claim_cur == os.O_RDONLY else "F_WRLCK"
                    self.dprint('DBG3', "Lock %s in a different process (F_SETLK, %s) start=%d len=%d" % (self.absfile, cio_lstr, self.lock_poffset, self.lock_len))
                    self.lexecobj.run(file_lock, fdsec, cio_type, self.absfile, self.lock_poffset, self.lock_len)
                except IOError as exerr:
                    fmsg = ", failed with %s" % exerr
                self.test(len(fmsg) == 0, "%s should succeed" % dmsg, failmsg=fmsg)

            if len(self.basename) > 0:
                if conflict_type == OP_RENAME and not target:
                    aname = self.absfile
                    fname = self.filename
                    self.get_filename()
                    self.absfile  = aname
                    self.filename = fname
                raise BaseName

            try:
                # Read/Write file
                fmsg = ""
                dmsg = "%s file on client holding delegation" % io_str.capitalize()
                self.dprint("DBG3", "%s [%s]" % (dmsg, self.absfile))
                if io_type == OPEN_READ:
                    os.read(fd, iosize)
                else:
                    os.write(fd, self.data_pattern(0, iosize, PATTERN))
            except OSError as exerr:
                fmsg = ", failed with %s" % exerr
            self.test(len(fmsg) == 0, "%s should succeed" % dmsg, failmsg=fmsg)
            self.delay_io()

            # Read same file from another client -- delegation should not be
            # recalled and delegation should be granted
            if other_read_deleg:
                # Other READ opens will not recall the delegation
                self.get_deleg_remote()

            # Absolute path name for remote file
            r_absfile = self.clientobj.abspath(self.filename)

            try:
                fmsg = ""
                if conflict_type == OP_SETATTR:
                    dmsg = "Change permissions on the file from another client"
                    self.dprint("DBG2", "%s to recall delegation [%s]" % (dmsg, r_absfile))
                    self.rexecobj.run(os.chmod, r_absfile, 0o777)
                elif conflict_type == OP_REMOVE:
                    dmsg = "Remove the file from another client"
                    self.dprint("DBG2", "%s to recall delegation [%s]" % (dmsg, r_absfile))
                    self.rexecobj.run(os.unlink, r_absfile)
                elif conflict_type == OP_RENAME:
                    if target:
                        fname = self.files[self.rfindex]
                        self.rfindex += 1
                        srcname = self.clientobj.abspath(fname)
                        dmsg = "Rename into the file (DST) from another client"
                        self.dprint("DBG2", "%s to recall delegation [%s -> %s]" % (dmsg, fname, self.filename))
                        self.rexecobj.run(os.rename, srcname, r_absfile)
                    else:
                        aname = self.absfile
                        fname = self.filename
                        self.get_filename()
                        dmsg = "Rename the file (SRC) from another client"
                        self.dprint("DBG2", "%s to recall delegation [%s -> %s]" % (dmsg, fname, self.filename))
                        newname = self.clientobj.abspath(self.filename)
                        self.absfile  = aname
                        self.filename = fname
                        self.rexecobj.run(os.rename, r_absfile, newname)
                elif conflict_type == OP_READ:
                    dmsg = "Read same file from another client"
                    self.dprint("DBG2", "%s to recall delegation [%s]" % (dmsg, r_absfile))
                    fdrd = self.rexecobj.run(os.open, r_absfile, os.O_RDONLY)
                    data = self.rexecobj.run(os.read, fdrd, 1024)
                    self.rexecobj.run(os.close, fdrd)
                elif self.truncate:
                    dmsg = "Write same file (truncate before writing) from another client"
                    self.dprint("DBG2", "%s to recall delegation [%s]" % (dmsg, r_absfile))
                    fdwr  = self.rexecobj.run(os.open, r_absfile, os.O_WRONLY|os.O_TRUNC)
                    count = self.rexecobj.run(os.write, fdwr, self.data_pattern(0, 1024, b"x"))
                    self.rexecobj.run(os.close, fdwr)
                else:
                    dmsg = "Write same file from another client"
                    self.dprint("DBG2", "%s to recall delegation [%s]" % (dmsg, r_absfile))
                    fdwr  = self.rexecobj.run(os.open, r_absfile, os.O_WRONLY|os.O_APPEND)
                    count = self.rexecobj.run(os.write, fdwr, self.data_pattern(0, 1024, b"X"))
                    self.rexecobj.run(os.close, fdwr)
            except OSError as exerr:
                fmsg = ", failed with %s" % exerr
            self.test(len(fmsg) == 0, "%s should succeed" % dmsg, failmsg=fmsg)

            try:
                fmsg = ""
                error = None
                dmsg = "%s file after conflicting operation" % io_str.capitalize()
                self.dprint("DBG3", "%s [%s]" % (dmsg, self.absfile))
                # Check for errors since the server may return an error
                # when the file has been removed by another client
                expfail = conflict_type == OP_REMOVE or (conflict_type == OP_RENAME and target)
                # Read/Write file
                if io_type == OPEN_READ:
                    os.read(fd, self.filesize - iosize)
                else:
                    os.write(fd, self.data_pattern(iosize, self.filesize - iosize, PATTERN))
                    # Flush data so if there is an error on a write it will
                    # happened here instead on the close
                    os.fdatasync(fd)
            except OSError as exerr:
                if expfail:
                    expected = errno.errorcode[errno.ESTALE]
                    error = errno.errorcode[exerr.errno]
                    fmsg =  ": expecting %s, got %s" % (expected, error)
                    self.dprint("DBG3", "%s returns: %s" % (dmsg, str(exerr)))
                    self.test(exerr.errno == errno.ESTALE, "%s may return an error" % dmsg, failmsg=fmsg)
                else:
                    fmsg = ", failed with %s" % exerr
            if not expfail:
                self.test(len(fmsg) == 0, "%s should succeed" % dmsg, failmsg=fmsg)
            elif error is None:
                self.test(len(fmsg) == 0, "%s may succeed" % dmsg, failmsg=fmsg)
            self.delay_io()
        except BaseName:
            pass
        except:
            self.test(False, traceback.format_exc())
        finally:
            if fd:
                # Close file
                self.dprint('DBG4', "Close %s" % self.absfile)
                os.close(fd)
            if self.fdko:
                # Close open owner file
                self.fdko.close()
            if fdsec:
                self.dprint('DBG4', "Close %s on second process" % self.absfile)
                self.lexecobj.run(os.close, fdsec)
            self.umount()
            if self.clientobj:
                self.clientobj.umount()
            self.trace_stop()

        try:
            if conflict_type == OP_REMOVE or (conflict_type == OP_RENAME and target):
                self.set_nfserr_list(
                    nfs3list=[NFS3ERR_NOENT, NFS3ERR_JUKEBOX],
                    nfs4list=[NFS4ERR_NOENT, NFS4ERR_DELAY, NFS4ERR_STALE, NFS4ERR_BAD_STATEID],
                )
            else:
                self.set_nfserr_list(
                    nfs3list=[NFS3ERR_NOENT, NFS3ERR_JUKEBOX],
                    nfs4list=[NFS4ERR_NOENT, NFS4ERR_DELAY],
                )
            self.trace_open()
            self.set_pktlist()

            # Find OPEN on second mount
            (open2fh, open2stid, deleg2stid) = self.find_open(filename=self.filename, src_ipaddr=self.clientobj.ipaddr, nfs_version=self.clientobj.nfs_version)
            self.pktt.rewind(0)

            # Find OPEN on main mount
            (open_fh, open_stid, deleg_stid) = self.find_open(filename=self.filename, deleg_type=deleg_type, src_ipaddr=self.client_ipaddr)
            self.verify_open(open_fh)
            self.test(deleg_stid != None, "%s delegation should be granted" % mode_str)
            save_index = self.pktt.get_index()
            open1_index = save_index
            if deleg_stid is None:
                # Delegation was not granted
                return

            if open2fh is None:
                open2fh = open_fh

            open_stateid = None
            if open_stid is not None:
                open_stateid = self.openreply.NFSop.stateid

            filehandles = [open_fh]
            self.find_layoutget(open_fh)
            (devcall, devreply, dslist) = self.find_getdeviceinfo()
            self.pktt.rewind(save_index)
            if self.layout:
                filehandles = self.layout['filehandles']

            fh1 = None
            if other_read_deleg:
                # Find OPEN (READ) call from the second client
                (fh1, op_stid1, deleg_stid1) = self.find_open(filename=self.filename, src_ipaddr=sipaddr, port=sport, proto=sproto, nfs_version=nfs_version)
                save_index = self.pktt.get_index() + 1

            # Find DELEGRETURN request and reply
            (delegreturncall, delegreturnreply) = self.find_nfs_op(OP_DELEGRETURN, src_ipaddr=self.client_ipaddr)
            if delegreturncall:
                delegreturn_index = delegreturncall.record.index
            self.pktt.rewind(save_index)

            # Find OPEN call from the second client
            src_other_client_str = "IP.dst == '%s'" % self.server_ipaddr
            if sproto in ("tcp", "udp"):
                src_other_client_str += " and %s.dst_port == %d" % (sproto.upper(), sport)
            src_other_client_str += " and IP.src == '%s' and " % sipaddr
            nfsver = self.match_nfs_version(nfs_version)
            conflict_match_str = src_other_client_str + nfsver + "NFS.argop == %d" % conflict_op
            if conflict_op == OP_OPEN:
                conflict_match_str += " and (NFS.claim.name == '%s' or" % self.filename
                conflict_match_str += " (NFS.fh == b'%s' and NFS.claim.claim == %d))" % (self.pktt.escape(open2fh), CLAIM_FH)
            elif target and conflict_type == OP_RENAME:
                conflict_match_str += " and NFS.newname == '%s'" % self.filename
            elif conflict_type in [OP_REMOVE, OP_RENAME]:
                conflict_match_str += " and NFS.name == '%s'" % self.filename
            elif conflict_op in [NFSPROC3_READ, NFSPROC3_WRITE]:
                if fh1 is None:
                    # Find the NFSv3 LOOKUP to get the file handle
                    (fh1, op_stid1, deleg_stid1) = self.find_open(filename=self.filename, src_ipaddr=sipaddr, port=sport, proto=sproto, nfs_version=nfs_version)
                expr = fh1 is not None
                self.test(expr, "LOOKUP should be sent from second client")
                if expr:
                   conflict_match_str += " and NFS.fh == b'%s'" % self.pktt.escape(fh1)

            silly_op = OP_RENAME if nfs_version > 3 else NFSPROC3_RENAME
            silly_match_str = src_other_client_str + nfsver + "NFS.argop == %d" % silly_op
            silly_match_str += " and NFS.name == '%s'" % self.filename

            opencall  = None
            openreply = None
            silly_rename = False
            if target and conflict_type == OP_RENAME:
                # Look for silly rename before looking for RENAME(DST)
                opencall, openreply = self.get_conflict_pkts(silly_op, silly_match_str)
                if opencall:
                    self.dprint('DBG2', "Silly RENAME found instead of RENAME(DST)")
                    silly_rename = True
                    conflict_op == silly_op
                    conflict_str = "RENAME"

            if opencall is None:
                # Get conflicting operation packets
                opencall, openreply = self.get_conflict_pkts(conflict_op, conflict_match_str)

            if opencall is None and conflict_type == OP_REMOVE and not (target and conflict_type == OP_RENAME):
                # REMOVE call was not found, look for a silly rename
                opencall, openreply = self.get_conflict_pkts(silly_op, silly_match_str)
                if opencall:
                    self.dprint('DBG2', "Silly RENAME found instead of REMOVE")
                    silly_rename = True
                    conflict_op == silly_op
                    conflict_str = "RENAME"

            if opencall is None:
                self.test(False, "%s should be sent from second client" % conflict_str)
                return
            conflict_index = opencall.record.index + 1
            self.pktt.rewind(conflict_index)

            # Boolean for pending test which should not recall the delegation
            read_write = deleg_type == OPEN_DELEGATE_READ and claim_cur == os.O_WRONLY

            if other_read_deleg:
                self.pktt.rewind(save_index)
                # Verify no CB_RECALL is sent to client under test
                (cbcall, cbreply) = self.find_nfs_op(OP_CB_RECALL, ipaddr=self.client_ipaddr, port=None, src_ipaddr=self.server_ipaddr, maxindex=conflict_index, first_call=True, nfs_version=None)
                self.test(cbcall is None, "CB_RECALL should not be sent to the client after a READ OPEN is received from a second client")
                if deleg_stid1 != None:
                    self.test(cbcall is None, "CB_RECALL should not be sent to the client after a second client is granted a READ delegation")
                self.pktt.rewind(conflict_index)

            lock_stid = None
            if lock:
                self.pktt.rewind(save_index)
                # Verify no CB_RECALL is sent to client under test
                (cbcall, cbreply) = self.find_nfs_op(OP_CB_RECALL, ipaddr=self.client_ipaddr, port=None, src_ipaddr=self.server_ipaddr, maxindex=conflict_index, first_call=True, nfs_version=None)
                self.test(cbcall is None, "%s delegation should not be recalled after locking the file" % mode_str)

                (lockcall, lockreply) = self.find_nfs_op(OP_LOCK, src_ipaddr=self.client_ipaddr, maxindex=conflict_index)
                if read_write:
                    self.test(lockcall, "LOCK should be sent when holding a %s delegation and the file is opened on the same client for writing" % mode_str)
                    if lockreply:
                        lock_stid = lockreply.NFSop.stateid.other
                else:
                    self.test(lockcall is None, "LOCK should not be sent when holding a %s delegation on the file" % mode_str)
                self.pktt.rewind(conflict_index)

            wrdexpr = False
            deleg2_stid = None
            open2_stid = None
            mode2_str = mode_str
            if read_write:
                # The client returns the delegation
                self.test(delegreturncall, "DELEGRETURN should be sent when the file is opened on the same client for writing")
                if delegreturncall:
                    expr = delegreturncall.NFSop.stateid.other == deleg_stid
                    self.test(expr, "DELEGRETURN should be sent with the %s delegation stateid" % mode_str)

                    # Find if there is an open after DELEGRETURN but before
                    # the conflicting operation
                    s_index = self.pktt.get_index()
                    self.pktt.rewind(delegreturncall.record.index)
                    (o_fh, o_stid, deleg2_stid) = self.find_open(filename=self.filename, claimfh=open_fh, anyclaim=True, src_ipaddr=self.client_ipaddr, maxindex=opencall.record.index)
                    if deleg2_stid is not None:
                        # Server returned a delegation for the new open
                        open2_stid = o_stid
                        deleg_type = self.openreply.NFSop.delegation.deleg_type
                        mode2_str  = deleg_str[self.openreply.NFSop.delegation.deleg_type]
                        wrdexpr = (self.opencall.NFSop.access == OPEN4_SHARE_ACCESS_WRITE and deleg_type == OPEN_DELEGATE_READ)

                        # Find the DELEGRETURN for this new delegation
                        (delegreturncall, delegreturnreply) = self.find_nfs_op(OP_DELEGRETURN, src_ipaddr=self.client_ipaddr)
                        if delegreturncall:
                            # Change index to second DELEGRETURN
                            delegreturn_index = delegreturncall.record.index
                    self.pktt.rewind(s_index)

            self.test(opencall, "%s should be sent from second client" % conflict_str)
            if conflict_op == OP_SETATTR:
                expr = opencall.NFSop.stateid.seqid == self.stateid_anonymous.seqid \
                   and opencall.NFSop.stateid.other == self.stateid_anonymous.other
                self.test(expr, "%s should be sent with the special anonymous stateid (0, 0)" % conflict_str)
            elif conflict_op == OP_REMOVE:
                expr = opencall.NFSop.name == self.filename
                self.test(expr, "%s should be sent with file holding the delegation as the name" % conflict_str)
            elif conflict_op == OP_RENAME:
                if target and not silly_rename:
                    expr = opencall.NFSop.newname == self.filename
                    self.test(expr, "%s should be sent with file holding the delegation as the target" % conflict_str)
                else:
                    expr = opencall.NFSop.name == self.filename
                    self.test(expr, "%s should be sent with file holding the delegation as the source" % conflict_str)

            # Find CB_RECALL sent to client under test
            (cbcall, cbreply) = self.find_nfs_op(OP_CB_RECALL, ipaddr=self.client_ipaddr, port=None, src_ipaddr=self.server_ipaddr, first_call=True, nfs_version=None)
            if read_write and not deleg2_stid:
                # The client has returned the delegation so no CB_RECALL
                self.test(cbcall is None, "CB_RECALL should not be sent to the client after a conflicting %s is received from a second client" % conflict_str)
                self.test(openreply, "%s reply should be sent to the second client" % conflict_str)
                # Verify I/O packets
                stid_type = LOCK_STID if lock and lock_stid else OPEN_STID
                stateid = lock_stid if stid_type == LOCK_STID else open_stid
                self.verify_io_packets(op_type, open_fh, stid_type, stateid, delegret=True)
            else:
                self.test(cbcall, "CB_RECALL should be sent to the client after a conflicting %s is received from a second client" % conflict_str)
            if cbcall is None:
                return
            cbrecall_index = self.pktt.get_index()
            if deleg2_stid is None:
                expr = cbcall.NFSop.stateid.other == deleg_stid
            else:
                expr = cbcall.NFSop.stateid.other == deleg2_stid
                mode_str = mode2_str
            self.test(expr, "CB_RECALL should recall %s delegation granted to client" % mode_str)
            self.test(cbreply, "CB_RECALL reply should be sent to the server")
            if cbreply:
                self.test(cbreply.NFSop.status == NFS4_OK, "CB_RECALL should return NFS4_OK")

            # Find OPEN sent from the client right before returning the delegation
            (fh, op_stid2, deleg_stid2) = self.find_open(filename=self.filename, deleg_stateid=deleg_stid, src_ipaddr=self.client_ipaddr, fh=open_fh)
            open_index = self.pktt.get_index()
            if fh is not None or (not wrdexpr and claim_cur in [os.O_RDONLY, os.O_WRONLY]):
                expr1 = deleg_type == OPEN_DELEGATE_READ  and claim_cur == os.O_RDONLY
                expr2 = deleg_type == OPEN_DELEGATE_WRITE and claim_cur == os.O_WRONLY
                # OPEN is only sent if main open and pending open are different
                if not (op_stid2 is None and (expr1 or expr2)):
                    self.test(op_stid2, "OPEN with CLAIM_DELEGATE_CUR is sent before returning the %s delegation after CB_RECALL" % mode_str)
            if fh is not None:
                self.test(op_stid2 == open_stid, "OPEN stateid should be the same as the original OPEN stateid")
                op_stateid = self.openreply.NFSop.stateid
                expr = op_stateid.seqid == open_stateid.seqid + 1
                self.test(expr, "OPEN stateid seqid should be increased by one from the original OPEN stateid")
                self.test(deleg_stid2 is None, "Delegation should not be granted when re-opening the file before returning the %s delegation after CB_RECALL" % mode_str)

            if deleg_type == OPEN_DELEGATE_WRITE:
                # Find out how much data has already been flushed right
                # before getting the CB_RECALL
                self.pktt.rewind(open1_index)
                nio = self.verify_io_requests(deleg_type, deleg_stid, filehandles, src_ipaddr=self.client_ipaddr, maxindex=cbrecall_index)
                if iosize > sum(self.test_counts):
                    # Not all data has been flushed,
                    # so find the WRITEs before DELEGRETURN
                    self.pktt.rewind(cbrecall_index)
                    nio = self.verify_io_requests(deleg_type, deleg_stid, filehandles, src_ipaddr=self.client_ipaddr, maxindex=open_index)
                    if nio > 0:
                        # Make this test optional since latest kernels do not
                        # flush the data before returning the delegation
                        self.test(True, "Client flushes written data before returning the WRITE delegation")
                else:
                    self.test(True, "Client has already flushed all written data before CB_RECALL")
                self.pktt.rewind(open_index)

            lock_stid2 = lock_stid
            lock_stid = None
            locker = None
            if lock and delegreturncall and not wrdexpr:
                # Verify the LOCK before DELEGRETURN
                lop_stid = lock_stid2 if lock_stid2 else open_stid if op_stid2 is None else op_stid2
                lock_stid = self.verify_lock(io_type, mode_str, lop_stid, self.lock_offset, open_index, delegreturn_index)
                if self.pktcall:
                    # Save lock owner
                    locker = self.pktcall.NFSop.locker
                if cio_type is not None:
                    # Verify the LOCK before DELEGRETURN -- second process
                    msg = " for the second process"
                    lop_stid = lop_stid if open2_stid is None else open2_stid
                    self.verify_lock(cio_type, mode_str, lop_stid, self.lock_poffset, open_index, delegreturn_index, msg=msg, locker=locker, lock_stid=lock_stid)

            self.test(delegreturncall, "DELEGRETURN should be sent")
            if delegreturncall is None:
                return
            deleg_stid = deleg_stid if deleg2_stid is None else deleg2_stid
            expr = delegreturncall.NFSop.stateid.other == deleg_stid
            self.test(expr, "DELEGRETURN should be sent with the stateid of %s delegation being recalled" % mode_str)

            self.pktt.rewind(delegreturn_index)

            # Find conflicting operation reply from the second client
            self.test(openreply != None, "%s reply should be sent to the second client after the %s delegation has been returned" % (conflict_str, mode_str))
            if openreply is None:
                return
            mds_stat = [NFS4_OK, NFS3_OK]
            ds_stat  = [NFS4_OK]
            if wrdexpr and lock_stid2:
                stid_type = LOCK_STID if lock and lock_stid2 else OPEN_STID
                stateid = lock_stid2 if stid_type == LOCK_STID else open_stid
            else:
                stid_type = LOCK_STID if lock and lock_stid else OPEN_STID
                stateid = lock_stid if stid_type == LOCK_STID else open_stid
            if conflict_op == OP_OPEN:
                self.test(openreply.NFSop.delegation.deleg_type == OPEN_DELEGATE_NONE, "Delegation should not be granted for the second client")
            elif conflict_type == OP_REMOVE or (conflict_type == OP_RENAME and target):
                ds_stat  = [NFS4ERR_STALE, NFS4ERR_BAD_STATEID]
                mds_stat = [NFS4ERR_STALE]

            # Verify I/O packets
            save_index = self.pktt.get_index()
            self.verify_io_packets(op_type, open_fh, stid_type, stateid, status=mds_stat, ds_status=ds_stat, delegret=True)

            # Find CLOSE request and reply
            self.verify_close(open_fh, open_stid, pindex=save_index)
        except Exception:
            self.test(False, traceback.format_exc())
        finally:
            self.pktt.close()

    def basic01_test(self):
        """Basic read delegation test"""
        self.basic_deleg_test(OPEN_READ)

    def basic02_test(self):
        """Basic write delegation test"""
        self.basic_deleg_test(OPEN_WRITE)

    def basic03_test(self):
        """Basic read delegation test with file stat"""
        self.basic_deleg_test(OPEN_READ, stat=True)

    def basic04_test(self):
        """Basic write delegation test with file stat"""
        self.basic_deleg_test(OPEN_WRITE, stat=True, nfiles=1)

    def basic05_test(self):
        """Basic read delegation test with file lock"""
        self.basic_deleg_test(OPEN_READ, lock=True)

    def basic06_test(self):
        """Basic write delegation test with file lock"""
        self.basic_deleg_test(OPEN_WRITE, lock=True)

    def basic07_test(self):
        """Basic write delegation test using RDWR open while reading"""
        self.basic_deleg_test(OPEN_RDWR, io_type=OPEN_READ)

    def basic08_test(self):
        """Basic write delegation test using RDWR open while writing"""
        self.basic_deleg_test(OPEN_RDWR, io_type=OPEN_WRITE)

    def basic09_test(self):
        """Basic write delegation test using RDWR open while reading with file stat"""
        self.basic_deleg_test(OPEN_RDWR, io_type=OPEN_READ, stat=True)

    def basic10_test(self):
        """Basic write delegation test using RDWR open while writing with file stat"""
        self.basic_deleg_test(OPEN_RDWR, io_type=OPEN_WRITE, stat=True, nfiles=1)

    def basic11_test(self):
        """Basic write delegation test using RDWR open while reading with file lock"""
        self.basic_deleg_test(OPEN_RDWR, io_type=OPEN_READ, lock=True)

    def basic12_test(self):
        """Basic write delegation test using RDWR open while writing with file lock"""
        self.basic_deleg_test(OPEN_RDWR, io_type=OPEN_WRITE, lock=True)

    def recall01_test(self):
        """Recall read delegation by writing from a second client"""
        self.recall_deleg_test(OPEN_READ)

    def recall02_test(self):
        """Recall write delegation by writing from a second client"""
        self.recall_deleg_test(OPEN_WRITE)

    def recall03_test(self):
        """Recall read delegation by writing from a second client with file lock"""
        self.recall_deleg_test(OPEN_READ, lock=True)

    def recall04_test(self):
        """Recall write delegation by writing from a second client with file lock"""
        self.recall_deleg_test(OPEN_WRITE, lock=True)

    def recall05_test(self):
        """Recall write delegation by reading from a second client"""
        self.recall_deleg_test(OPEN_WRITE, conflict_type=OP_READ)

    def recall06_test(self):
        """Recall write delegation by reading from a second client with file lock"""
        self.recall_deleg_test(OPEN_WRITE, conflict_type=OP_READ, lock=True)

    def recall07_test(self):
        """Recall read delegation by changing the permissions to the file"""
        self.recall_deleg_test(OPEN_READ, conflict_type=OP_SETATTR)

    def recall08_test(self):
        """Recall write delegation by changing the permissions to the file"""
        self.recall_deleg_test(OPEN_WRITE, conflict_type=OP_SETATTR)

    def recall09_test(self):
        """Recall read delegation by changing the permissions to the file with file lock"""
        self.recall_deleg_test(OPEN_READ, conflict_type=OP_SETATTR, lock=True)

    def recall10_test(self):
        """Recall write delegation by changing the permissions to the file with file lock"""
        self.recall_deleg_test(OPEN_WRITE, conflict_type=OP_SETATTR, lock=True)

    def recall11_test(self):
        """Recall read delegation by removing the file"""
        self.recall_deleg_test(OPEN_READ, conflict_type=OP_REMOVE, nfiles=1)

    def recall12_test(self):
        """Recall write delegation by removing the file"""
        self.recall_deleg_test(OPEN_WRITE, conflict_type=OP_REMOVE, nfiles=1)

    def recall13_test(self):
        """Recall read delegation by removing the file with file lock"""
        self.recall_deleg_test(OPEN_READ, conflict_type=OP_REMOVE, nfiles=1, lock=True)

    def recall14_test(self):
        """Recall write delegation by removing the file with file lock"""
        self.recall_deleg_test(OPEN_WRITE, conflict_type=OP_REMOVE, nfiles=1, lock=True)

    def recall15_test(self):
        """Recall read delegation by renaming the file"""
        self.recall_deleg_test(OPEN_READ, conflict_type=OP_RENAME, nfiles=1)

    def recall16_test(self):
        """Recall write delegation by renaming the file"""
        self.recall_deleg_test(OPEN_WRITE, conflict_type=OP_RENAME, nfiles=1)

    def recall17_test(self):
        """Recall read delegation by renaming the file with file lock"""
        self.recall_deleg_test(OPEN_READ, conflict_type=OP_RENAME, nfiles=1, lock=True)

    def recall18_test(self):
        """Recall write delegation by renaming the file with file lock"""
        self.recall_deleg_test(OPEN_WRITE, conflict_type=OP_RENAME, nfiles=1, lock=True)

    def recall19_test(self):
        """Recall read delegation by renaming into the file"""
        self.recall_deleg_test(OPEN_READ, conflict_type=OP_RENAME, nfiles=2, target=True)

    def recall20_test(self):
        """Recall write delegation by renaming into the file"""
        self.recall_deleg_test(OPEN_WRITE, conflict_type=OP_RENAME, nfiles=2, target=True)

    def recall21_test(self):
        """Recall read delegation by renaming into the file with file lock"""
        self.recall_deleg_test(OPEN_READ, conflict_type=OP_RENAME, nfiles=2, target=True, lock=True)

    def recall22_test(self):
        """Recall write delegation by renaming into the file with file lock"""
        self.recall_deleg_test(OPEN_WRITE, conflict_type=OP_RENAME, nfiles=2, target=True, lock=True)

    def recall23_test(self):
        """Recall read delegation by writing from a second client with file lock,
           having a pending read open"""
        self.recall_deleg_test(OPEN_READ, claim_cur=os.O_RDONLY, lock=True)

    def recall24_test(self):
        """Recall read delegation by writing from a second client with file lock,
           having a pending write open.
           Delegation is returned by the client when the second open is done so
           there is no delegation recall"""
        self.recall_deleg_test(OPEN_READ, claim_cur=os.O_WRONLY, lock=True)

    def recall25_test(self):
        """Recall write delegation by writing from a second client with file lock,
           having a pending read open"""
        self.recall_deleg_test(OPEN_WRITE, claim_cur=os.O_RDONLY, lock=True)

    def recall26_test(self):
        """Recall write delegation by writing from a second client with file lock,
           having a pending write open"""
        self.recall_deleg_test(OPEN_WRITE, claim_cur=os.O_WRONLY, lock=True)

    def recall27_test(self):
        """Recall write delegation by reading from a second client using RDWR
           open while reading"""
        self.recall_deleg_test(OPEN_RDWR, io_type=OPEN_READ, conflict_type=OP_READ)

    def recall28_test(self):
        """Recall write delegation by reading from a second client using RDWR
           open while writing"""
        self.recall_deleg_test(OPEN_RDWR, io_type=OPEN_WRITE, conflict_type=OP_READ)

    def recall29_test(self):
        """Recall write delegation by writing from a second client using RDWR
           open while reading"""
        self.recall_deleg_test(OPEN_RDWR, io_type=OPEN_READ, conflict_type=OP_WRITE)

    def recall30_test(self):
        """Recall write delegation by writing from a second client using RDWR
           open while writing"""
        self.recall_deleg_test(OPEN_RDWR, io_type=OPEN_WRITE, conflict_type=OP_WRITE)

    def recall31_test(self):
        """Recall write delegation by reading from a second client using RDWR
           open while reading with file lock"""
        self.recall_deleg_test(OPEN_RDWR, io_type=OPEN_READ, conflict_type=OP_READ, lock=True)

    def recall32_test(self):
        """Recall write delegation by reading from a second client using RDWR
           open while writing with file lock"""
        self.recall_deleg_test(OPEN_RDWR, io_type=OPEN_WRITE, conflict_type=OP_READ, lock=True)

    def recall33_test(self):
        """Recall write delegation by writing from a second client using RDWR
           open while reading with file lock"""
        self.recall_deleg_test(OPEN_RDWR, io_type=OPEN_READ, conflict_type=OP_WRITE, lock=True)

    def recall34_test(self):
        """Recall write delegation by writing from a second client using RDWR
           open while writing with file lock"""
        self.recall_deleg_test(OPEN_RDWR, io_type=OPEN_WRITE, conflict_type=OP_WRITE, lock=True)

    def recall35_test(self):
        """Recall write delegation by changing the permissions to the file
           from a second client using RDWR open while reading"""
        self.recall_deleg_test(OPEN_RDWR, io_type=OPEN_READ, conflict_type=OP_SETATTR)

    def recall36_test(self):
        """Recall write delegation by changing the permissions to the file
           from a second client using RDWR open while writing"""
        self.recall_deleg_test(OPEN_RDWR, io_type=OPEN_WRITE, conflict_type=OP_SETATTR)

    def recall37_test(self):
        """Recall write delegation by changing the permissions to the file
           from a second client using RDWR open while reading with file lock"""
        self.recall_deleg_test(OPEN_RDWR, io_type=OPEN_READ, conflict_type=OP_SETATTR, lock=True)

    def recall38_test(self):
        """Recall write delegation by changing the permissions to the file
           from a second client using RDWR open while writing with file lock"""
        self.recall_deleg_test(OPEN_RDWR, io_type=OPEN_WRITE, conflict_type=OP_SETATTR, lock=True)

    def recall39_test(self):
        """Recall write delegation by removing the file from a second client
           using RDWR open while reading"""
        self.recall_deleg_test(OPEN_RDWR, io_type=OPEN_READ, conflict_type=OP_REMOVE, nfiles=1)

    def recall40_test(self):
        """Recall write delegation by removing the file from a second client
           using RDWR open while writing"""
        self.recall_deleg_test(OPEN_RDWR, io_type=OPEN_WRITE, conflict_type=OP_REMOVE, nfiles=1)

    def recall41_test(self):
        """Recall write delegation by removing the file from a second client
           using RDWR open while reading with file lock"""
        self.recall_deleg_test(OPEN_RDWR, io_type=OPEN_READ, conflict_type=OP_REMOVE, nfiles=1, lock=True)

    def recall42_test(self):
        """Recall write delegation by removing the file from a second client
           using RDWR open while writing with file lock"""
        self.recall_deleg_test(OPEN_RDWR, io_type=OPEN_WRITE, conflict_type=OP_REMOVE, nfiles=1, lock=True)

    def recall43_test(self):
        """Recall write delegation by renaming the file from a second client
           using RDWR open while reading"""
        self.recall_deleg_test(OPEN_RDWR, io_type=OPEN_READ, conflict_type=OP_RENAME, nfiles=1)

    def recall44_test(self):
        """Recall write delegation by renaming the file from a second client
           using RDWR open while writing"""
        self.recall_deleg_test(OPEN_RDWR, io_type=OPEN_WRITE, conflict_type=OP_RENAME, nfiles=1)

    def recall45_test(self):
        """Recall write delegation by renaming the file from a second client
           using RDWR open while reading with file lock"""
        self.recall_deleg_test(OPEN_RDWR, io_type=OPEN_READ, conflict_type=OP_RENAME, nfiles=1, lock=True)

    def recall46_test(self):
        """Recall write delegation by renaming the file from a second client
           using RDWR open while writing with file lock"""
        self.recall_deleg_test(OPEN_RDWR, io_type=OPEN_WRITE, conflict_type=OP_RENAME, nfiles=1, lock=True)

    def recall47_test(self):
        """Recall write delegation by renaming the file from a second client
           using RDWR open while reading"""
        self.recall_deleg_test(OPEN_RDWR, io_type=OPEN_READ, conflict_type=OP_RENAME, nfiles=2, target=True)

    def recall48_test(self):
        """Recall write delegation by renaming the file from a second client
           using RDWR open while writing"""
        self.recall_deleg_test(OPEN_RDWR, io_type=OPEN_WRITE, conflict_type=OP_RENAME, nfiles=2, target=True)

    def recall49_test(self):
        """Recall write delegation by renaming the file from a second client
           using RDWR open while reading with file lock"""
        self.recall_deleg_test(OPEN_RDWR, io_type=OPEN_READ, conflict_type=OP_RENAME, nfiles=2, target=True, lock=True)

    def recall50_test(self):
        """Recall write delegation by renaming the file from a second client
           using RDWR open while writing with file lock"""
        self.recall_deleg_test(OPEN_RDWR, io_type=OPEN_WRITE, conflict_type=OP_RENAME, nfiles=2, target=True, lock=True)

    def recall51_test(self):
        """Recall write delegation by writing from a second client using RDWR
           open while reading with file lock, having a pending read open"""
        self.recall_deleg_test(OPEN_RDWR, io_type=OPEN_READ, claim_cur=os.O_RDONLY, lock=True)

    def recall52_test(self):
        """Recall write delegation by writing from a second client using RDWR
           open while reading with file lock, having a pending write open"""
        self.recall_deleg_test(OPEN_RDWR, io_type=OPEN_READ, claim_cur=os.O_WRONLY, lock=True)

    def recall53_test(self):
        """Recall write delegation by writing from a second client using RDWR
           open while writing with file lock, having a pending read open"""
        self.recall_deleg_test(OPEN_RDWR, io_type=OPEN_WRITE, claim_cur=os.O_RDONLY, lock=True)

    def recall54_test(self):
        """Recall write delegation by writing from a second client using RDWR
           open while writing with file lock, having a pending write open"""
        self.recall_deleg_test(OPEN_RDWR, io_type=OPEN_WRITE, claim_cur=os.O_WRONLY, lock=True)

################################################################################
# Entry point
x = DelegTest(usage=USAGE, testnames=TESTNAMES, testgroups=TESTGROUPS, sid=SCRIPT_ID)

try:
    x.setup()

    # Run all the tests
    x.run_tests()
except Exception:
    x.test(False, traceback.format_exc())
finally:
    if x.clientobj is not None and x.clientobj.mounted:
        # Unmount server on remote client
        x.clientobj.umount()
    x.cleanup()
    x.exit()
