tests.system.aiplatform.test_vizier.TestVizier: test_vizier_trial_deletion failed

[flaky-bot]

Note: #1962 was also for this test, but it was closed more than 10 days ago. So, I didn't mark it flaky.

---

commit: 921bfc0
buildURL: Build Status, Sponge
status: failed

Test output

args = (name: "projects/580378083368/locations/us-central1/studies/501554175607/operations/501554175607_halio_test_worker_1"
,)
kwargs = {'metadata': [('x-goog-request-params', 'name=projects/580378083368/locations/us-central1/studies/501554175607/operati...501554175607_halio_test_worker_1'), ('x-goog-api-client', 'gl-python/3.8.13 grpc/1.47.0 gax/1.32.0')], 'timeout': 20.0}
@six.wraps(callable_)
def error_remapped_callable(*args, **kwargs):
    try:


      return callable_(*args, **kwargs)


.nox/system-3-8/lib/python3.8/site-packages/google/api_core/grpc_helpers.py:67:

self = <grpc._channel._UnaryUnaryMultiCallable object at 0x7f474d4f3be0>

request = name: "projects/580378083368/locations/us-central1/studies/501554175607/operations/501554175607_halio_test_worker_1"
timeout = 20.0

metadata = [('x-goog-request-params', 'name=projects/580378083368/locations/us-central1/studies/501554175607/operations/501554175607_halio_test_worker_1'), ('x-goog-api-client', 'gl-python/3.8.13 grpc/1.47.0 gax/1.32.0')]

credentials = None, wait_for_ready = None, compression = None
def __call__(self,
             request,
             timeout=None,
             metadata=None,
             credentials=None,
             wait_for_ready=None,
             compression=None):
    state, call, = self._blocking(request, timeout, metadata, credentials,
                                  wait_for_ready, compression)


  return _end_unary_response_blocking(state, call, False, None)


.nox/system-3-8/lib/python3.8/site-packages/grpc/_channel.py:946:

state = <grpc._channel._RPCState object at 0x7f474d4f2ee0>

call = <grpc._cython.cygrpc.SegregatedCall object at 0x7f474d851b40>

with_call = False, deadline = None
def _end_unary_response_blocking(state, call, with_call, deadline):
    if state.code is grpc.StatusCode.OK:
        if with_call:
            rendezvous = _MultiThreadedRendezvous(state, call, None, deadline)
            return state.response, rendezvous
        else:
            return state.response
    else:


      raise _InactiveRpcError(state)


E           grpc._channel._InactiveRpcError: <_InactiveRpcError of RPC that terminated with:

E           	status = StatusCode.DEADLINE_EXCEEDED

E           	details = "Deadline Exceeded"

E           	debug_error_string = "{"created":"@1683037939.564954716","description":"Deadline Exceeded","file":"src/core/ext/filters/deadline/deadline_filter.cc","file_line":81,"grpc_status":4}"

E           >
.nox/system-3-8/lib/python3.8/site-packages/grpc/_channel.py:849: _InactiveRpcError
The above exception was the direct cause of the following exception:
self = <tests.system.aiplatform.test_vizier.TestVizier object at 0x7f474e081280>

shared_state = {'resources': [<google.cloud.aiplatform.vizier.study.Study object at 0x7f474d46eaf0>

resource name: projects/580378083368/locations/us-central1/studies/1997269120179]}
def test_vizier_trial_deletion(self, shared_state):
    aiplatform.init(
        project=e2e_base._PROJECT,
        location=e2e_base._LOCATION,
    )
    sc = pyvizier.StudyConfig()
    sc.algorithm = pyvizier.Algorithm.RANDOM_SEARCH
    sc.metric_information.append(
        pyvizier.MetricInformation(
            name="pr-auc", goal=pyvizier.ObjectiveMetricGoal.MAXIMIZE
        )
    )
    root = sc.search_space.select_root()
    root.add_float_param(
        "learning_rate", 0.00001, 1.0, scale_type=pyvizier.ScaleType.LINEAR
    )
    root.add_categorical_param("optimizer", ["adagrad", "adam", "experimental"])
    sc.automated_stopping_config = (
        pyvizier.AutomatedStoppingConfig.decay_curve_stopping_config(use_steps=True)
    )

    study = Study.create_or_load(display_name=self._temp_prefix, problem=sc)


  trials = study.suggest(count=1, worker="halio_test_worker")


tests/system/aiplatform/test_vizier.py:102:

google/cloud/aiplatform/vizier/study.py:286: in suggest

trials = suggest_trials_lro.result()

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:130: in result

self._blocking_poll(timeout=timeout, **kwargs)

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:108: in blocking_poll

retry(self._done_or_raise)(**kwargs)

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:286: in retry_wrapped_func

return retry_target(

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:189: in retry_target

return target()

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:86: in _done_or_raise

if not self.done(**kwargs):

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/operation.py:170: in done

self._refresh_and_update(retry)

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/operation.py:158: in _refresh_and_update

self._operation = self._refresh(retry=retry)

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/operations_v1/operations_client.py:142: in get_operation

return self._get_operation(

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/gapic_v1/method.py:145: in call

return wrapped_func(*args, **kwargs)

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:286: in retry_wrapped_func

return retry_target(

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:189: in retry_target

return target()

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/timeout.py:214: in func_with_timeout

return func(*args, **kwargs)

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/grpc_helpers.py:69: in error_remapped_callable

six.raise_from(exceptions.from_grpc_error(exc), exc)

value = None

from_value = <_InactiveRpcError of RPC that terminated with:

status = StatusCode.DEADLINE_EXCEEDED

details = "Deadline Exceeded"

...ption":"Deadline Exceeded","file":"src/core/ext/filters/deadline/deadline_filter.cc","file_line":81,"grpc_status":4}"



???

E   google.api_core.exceptions.DeadlineExceeded: 504 Deadline Exceeded

:3: DeadlineExceeded

[flaky-bot]

Test passed for commit 43488fe (Build Status, Sponge)! Closing this issue.