diff --git a/deps/build.jl b/deps/build.jl index 1a104aed..caabe43d 100644 --- a/deps/build.jl +++ b/deps/build.jl @@ -1,8 +1,8 @@ using PyCall using Conda -const cur_version = "1.10.0" -const cur_py_version = "1.10.0" +const cur_version = "1.12.0" +const cur_py_version = "1.12.0" ############################ diff --git a/src/TensorFlow.jl b/src/TensorFlow.jl index 69b76c45..6147526c 100644 --- a/src/TensorFlow.jl +++ b/src/TensorFlow.jl @@ -129,6 +129,12 @@ tf_versioninfo using Distributed +# Load these packages here so they are available to the additional +# process spawned in 'load_python_process. Arslan thinks that will +# work for now. +using MacroTools +using PyCall + const pyproc = Ref(0) function deallocator(data, len, arg) diff --git a/src/protobufs/allocation_description_pb.jl b/src/protobufs/allocation_description_pb.jl index 98801b59..147045bd 100644 --- a/src/protobufs/allocation_description_pb.jl +++ b/src/protobufs/allocation_description_pb.jl @@ -1,5 +1,4 @@ # syntax: proto3 -using Compat using ProtoBuf import ProtoBuf.meta diff --git a/src/protobufs/any_pb.jl b/src/protobufs/any_pb.jl index 116ab3df..859c3ce1 100644 --- a/src/protobufs/any_pb.jl +++ b/src/protobufs/any_pb.jl @@ -1,5 +1,4 @@ # syntax: proto3 -using Compat using ProtoBuf import ProtoBuf.meta diff --git a/src/protobufs/attr_value_pb.jl b/src/protobufs/attr_value_pb.jl index 9cf44b3d..169906f4 100644 --- a/src/protobufs/attr_value_pb.jl +++ b/src/protobufs/attr_value_pb.jl @@ -1,5 +1,4 @@ # syntax: proto3 -using Compat using ProtoBuf import ProtoBuf.meta diff --git a/src/protobufs/checkpointable_object_graph_pb.jl b/src/protobufs/checkpointable_object_graph_pb.jl index 0bf3d0a2..c7a263a1 100644 --- a/src/protobufs/checkpointable_object_graph_pb.jl +++ b/src/protobufs/checkpointable_object_graph_pb.jl @@ -1,5 +1,4 @@ # syntax: proto3 -using Compat using ProtoBuf import ProtoBuf.meta @@ -13,6 +12,7 @@ mutable struct CheckpointableObjectGraph_CheckpointableObject_SerializedTensor < name::AbstractString full_name::AbstractString checkpoint_key::AbstractString + optional_restore::Bool CheckpointableObjectGraph_CheckpointableObject_SerializedTensor(; kwargs...) = (o=new(); fillunset(o); isempty(kwargs) || ProtoBuf._protobuild(o, kwargs); o) end #mutable struct CheckpointableObjectGraph_CheckpointableObject_SerializedTensor diff --git a/src/protobufs/cluster_pb.jl b/src/protobufs/cluster_pb.jl index 56cc25e5..9442c764 100644 --- a/src/protobufs/cluster_pb.jl +++ b/src/protobufs/cluster_pb.jl @@ -1,5 +1,4 @@ # syntax: proto3 -using Compat using ProtoBuf import ProtoBuf.meta diff --git a/src/protobufs/config_pb.jl b/src/protobufs/config_pb.jl index 5f30469b..41bb2ab7 100644 --- a/src/protobufs/config_pb.jl +++ b/src/protobufs/config_pb.jl @@ -1,5 +1,4 @@ # syntax: proto3 -using Compat using ProtoBuf import ProtoBuf.meta @@ -14,6 +13,7 @@ mutable struct GPUOptions_Experimental <: ProtoType virtual_devices::Base.Vector{GPUOptions_Experimental_VirtualDevices} use_unified_memory::Bool num_dev_to_dev_copy_streams::Int32 + collective_ring_order::AbstractString GPUOptions_Experimental(; kwargs...) = (o=new(); fillunset(o); isempty(kwargs) || ProtoBuf._protobuild(o, kwargs); o) end #mutable struct GPUOptions_Experimental @@ -83,6 +83,8 @@ end #mutable struct ThreadPoolOptionProto mutable struct RPCOptions <: ProtoType use_rpc_for_inprocess_master::Bool + compression_algorithm::AbstractString + compression_level::Int32 RPCOptions(; kwargs...) = (o=new(); fillunset(o); isempty(kwargs) || ProtoBuf._protobuild(o, kwargs); o) end #mutable struct RPCOptions @@ -95,9 +97,12 @@ end #mutable struct ConfigProto_DeviceCountEntry (mapentry) mutable struct ConfigProto_Experimental <: ProtoType collective_group_leader::AbstractString executor_type::AbstractString + recv_buf_max_chunk::Int32 + use_numa_affinity::Bool + collective_deterministic_sequential_execution::Bool ConfigProto_Experimental(; kwargs...) = (o=new(); fillunset(o); isempty(kwargs) || ProtoBuf._protobuild(o, kwargs); o) end #mutable struct ConfigProto_Experimental -const __fnum_ConfigProto_Experimental = Int[1,3] +const __fnum_ConfigProto_Experimental = Int[1,3,4,5,6] meta(t::Type{ConfigProto_Experimental}) = meta(t, ProtoBuf.DEF_REQ, __fnum_ConfigProto_Experimental, ProtoBuf.DEF_VAL, true, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES, ProtoBuf.DEF_ONEOFS, ProtoBuf.DEF_ONEOF_NAMES, ProtoBuf.DEF_FIELD_TYPES) mutable struct ConfigProto <: ProtoType diff --git a/src/protobufs/control_flow_pb.jl b/src/protobufs/control_flow_pb.jl index e0d6a791..c98fe684 100644 --- a/src/protobufs/control_flow_pb.jl +++ b/src/protobufs/control_flow_pb.jl @@ -1,5 +1,4 @@ # syntax: proto3 -using Compat using ProtoBuf import ProtoBuf.meta diff --git a/src/protobufs/cost_graph_pb.jl b/src/protobufs/cost_graph_pb.jl index 706056ff..8e071288 100644 --- a/src/protobufs/cost_graph_pb.jl +++ b/src/protobufs/cost_graph_pb.jl @@ -1,5 +1,4 @@ # syntax: proto3 -using Compat using ProtoBuf import ProtoBuf.meta diff --git a/src/protobufs/critical_section_pb.jl b/src/protobufs/critical_section_pb.jl index c65d844b..64840dcd 100644 --- a/src/protobufs/critical_section_pb.jl +++ b/src/protobufs/critical_section_pb.jl @@ -1,5 +1,4 @@ # syntax: proto3 -using Compat using ProtoBuf import ProtoBuf.meta diff --git a/src/protobufs/debug_pb.jl b/src/protobufs/debug_pb.jl index 74d98b5c..3ecedab2 100644 --- a/src/protobufs/debug_pb.jl +++ b/src/protobufs/debug_pb.jl @@ -1,5 +1,4 @@ # syntax: proto3 -using Compat using ProtoBuf import ProtoBuf.meta diff --git a/src/protobufs/device_attributes_pb.jl b/src/protobufs/device_attributes_pb.jl index 857821b8..eb495f75 100644 --- a/src/protobufs/device_attributes_pb.jl +++ b/src/protobufs/device_attributes_pb.jl @@ -1,5 +1,4 @@ # syntax: proto3 -using Compat using ProtoBuf import ProtoBuf.meta diff --git a/src/protobufs/device_properties_pb.jl b/src/protobufs/device_properties_pb.jl index 9a790280..2502b691 100644 --- a/src/protobufs/device_properties_pb.jl +++ b/src/protobufs/device_properties_pb.jl @@ -1,5 +1,4 @@ # syntax: proto3 -using Compat using ProtoBuf import ProtoBuf.meta diff --git a/src/protobufs/eager_service_pb.jl b/src/protobufs/eager_service_pb.jl index 1e942409..766abde1 100644 --- a/src/protobufs/eager_service_pb.jl +++ b/src/protobufs/eager_service_pb.jl @@ -1,5 +1,4 @@ # syntax: proto3 -using Compat using ProtoBuf import ProtoBuf.meta import ._ProtoBuf_Top_.tensorflow diff --git a/src/protobufs/error_codes_pb.jl b/src/protobufs/error_codes_pb.jl index 62f3159d..1e113d7b 100644 --- a/src/protobufs/error_codes_pb.jl +++ b/src/protobufs/error_codes_pb.jl @@ -1,5 +1,4 @@ # syntax: proto3 -using Compat using ProtoBuf import ProtoBuf.meta diff --git a/src/protobufs/function_pb.jl b/src/protobufs/function_pb.jl index f0b37127..349cc014 100644 --- a/src/protobufs/function_pb.jl +++ b/src/protobufs/function_pb.jl @@ -1,5 +1,4 @@ # syntax: proto3 -using Compat using ProtoBuf import ProtoBuf.meta diff --git a/src/protobufs/google.jl b/src/protobufs/google.jl index 8c5eb041..3aa19652 100644 --- a/src/protobufs/google.jl +++ b/src/protobufs/google.jl @@ -1,8 +1,6 @@ module google - using Compat const _ProtoBuf_Top_ = @static isdefined(parentmodule(@__MODULE__), :_ProtoBuf_Top_) ? (parentmodule(@__MODULE__))._ProtoBuf_Top_ : parentmodule(@__MODULE__) module protobuf - using Compat const _ProtoBuf_Top_ = @static isdefined(parentmodule(@__MODULE__), :_ProtoBuf_Top_) ? (parentmodule(@__MODULE__))._ProtoBuf_Top_ : parentmodule(@__MODULE__) include("any_pb.jl") end diff --git a/src/protobufs/graph_pb.jl b/src/protobufs/graph_pb.jl index 690e335b..b2acee13 100644 --- a/src/protobufs/graph_pb.jl +++ b/src/protobufs/graph_pb.jl @@ -1,5 +1,4 @@ # syntax: proto3 -using Compat using ProtoBuf import ProtoBuf.meta diff --git a/src/protobufs/master_pb.jl b/src/protobufs/master_pb.jl index 1b5a9301..7918632d 100644 --- a/src/protobufs/master_pb.jl +++ b/src/protobufs/master_pb.jl @@ -1,5 +1,4 @@ # syntax: proto3 -using Compat using ProtoBuf import ProtoBuf.meta import ._ProtoBuf_Top_.tensorflow diff --git a/src/protobufs/master_service_pb.jl b/src/protobufs/master_service_pb.jl index 5e041449..fd2c45d5 100644 --- a/src/protobufs/master_service_pb.jl +++ b/src/protobufs/master_service_pb.jl @@ -1,5 +1,4 @@ # syntax: proto3 -using Compat using ProtoBuf import ProtoBuf.meta import ._ProtoBuf_Top_.tensorflow diff --git a/src/protobufs/meta_graph_pb.jl b/src/protobufs/meta_graph_pb.jl index 51acd3b3..687739f8 100644 --- a/src/protobufs/meta_graph_pb.jl +++ b/src/protobufs/meta_graph_pb.jl @@ -1,5 +1,4 @@ # syntax: proto3 -using Compat using ProtoBuf import ProtoBuf.meta import ProtoBuf.google.protobuf diff --git a/src/protobufs/named_tensor_pb.jl b/src/protobufs/named_tensor_pb.jl index 6909110c..13b335a2 100644 --- a/src/protobufs/named_tensor_pb.jl +++ b/src/protobufs/named_tensor_pb.jl @@ -1,5 +1,4 @@ # syntax: proto3 -using Compat using ProtoBuf import ProtoBuf.meta diff --git a/src/protobufs/node_def_pb.jl b/src/protobufs/node_def_pb.jl index 1f8c85d8..a101bab8 100644 --- a/src/protobufs/node_def_pb.jl +++ b/src/protobufs/node_def_pb.jl @@ -1,5 +1,4 @@ # syntax: proto3 -using Compat using ProtoBuf import ProtoBuf.meta @@ -9,14 +8,20 @@ mutable struct NodeDef_AttrEntry <: ProtoType NodeDef_AttrEntry(; kwargs...) = (o=new(); fillunset(o); isempty(kwargs) || ProtoBuf._protobuild(o, kwargs); o) end #mutable struct NodeDef_AttrEntry (mapentry) +mutable struct NodeDef_ExperimentalDebugInfo <: ProtoType + original_node_names::Base.Vector{AbstractString} + NodeDef_ExperimentalDebugInfo(; kwargs...) = (o=new(); fillunset(o); isempty(kwargs) || ProtoBuf._protobuild(o, kwargs); o) +end #mutable struct NodeDef_ExperimentalDebugInfo + mutable struct NodeDef <: ProtoType name::AbstractString op::AbstractString input::Base.Vector{AbstractString} device::AbstractString attr::Base.Dict{AbstractString,AttrValue} # map entry + experimental_debug_info::NodeDef_ExperimentalDebugInfo NodeDef(; kwargs...) = (o=new(); fillunset(o); isempty(kwargs) || ProtoBuf._protobuild(o, kwargs); o) end #mutable struct NodeDef -export NodeDef_AttrEntry, NodeDef +export NodeDef_AttrEntry, NodeDef_ExperimentalDebugInfo, NodeDef # mapentries: "NodeDef_AttrEntry" => ("AbstractString", "AttrValue") diff --git a/src/protobufs/op_def_pb.jl b/src/protobufs/op_def_pb.jl index b29c8fa0..fce211e8 100644 --- a/src/protobufs/op_def_pb.jl +++ b/src/protobufs/op_def_pb.jl @@ -1,5 +1,4 @@ # syntax: proto3 -using Compat using ProtoBuf import ProtoBuf.meta diff --git a/src/protobufs/queue_runner_pb.jl b/src/protobufs/queue_runner_pb.jl index 70a457e7..b116687e 100644 --- a/src/protobufs/queue_runner_pb.jl +++ b/src/protobufs/queue_runner_pb.jl @@ -1,5 +1,4 @@ # syntax: proto3 -using Compat using ProtoBuf import ProtoBuf.meta import ._ProtoBuf_Top_.tensorflow diff --git a/src/protobufs/replay_log_pb.jl b/src/protobufs/replay_log_pb.jl index cb30c5cb..f636d4d0 100644 --- a/src/protobufs/replay_log_pb.jl +++ b/src/protobufs/replay_log_pb.jl @@ -1,5 +1,4 @@ # syntax: proto3 -using Compat using ProtoBuf import ProtoBuf.meta diff --git a/src/protobufs/resource_handle_pb.jl b/src/protobufs/resource_handle_pb.jl index 4c47da93..507972ba 100644 --- a/src/protobufs/resource_handle_pb.jl +++ b/src/protobufs/resource_handle_pb.jl @@ -1,5 +1,4 @@ # syntax: proto3 -using Compat using ProtoBuf import ProtoBuf.meta diff --git a/src/protobufs/rewriter_config_pb.jl b/src/protobufs/rewriter_config_pb.jl index 6d66e150..9ea7f88a 100644 --- a/src/protobufs/rewriter_config_pb.jl +++ b/src/protobufs/rewriter_config_pb.jl @@ -1,5 +1,4 @@ # syntax: proto3 -using Compat using ProtoBuf import ProtoBuf.meta @@ -73,13 +72,15 @@ mutable struct RewriterConfig <: ProtoType min_graph_nodes::Int32 memory_optimization::Int32 memory_optimizer_target_node_name_scope::AbstractString + meta_optimizer_timeout_ms::Int64 auto_parallel::AutoParallelOptions + fail_on_optimizer_errors::Bool scoped_allocator_opts::ScopedAllocatorOptions optimizers::Base.Vector{AbstractString} custom_optimizers::Base.Vector{RewriterConfig_CustomGraphOptimizer} RewriterConfig(; kwargs...) = (o=new(); fillunset(o); isempty(kwargs) || ProtoBuf._protobuild(o, kwargs); o) end #mutable struct RewriterConfig -const __fnum_RewriterConfig = Int[1,3,13,14,7,8,9,10,11,2,15,18,19,12,17,4,6,5,16,100,200] +const __fnum_RewriterConfig = Int[1,3,13,14,7,8,9,10,11,2,15,18,19,12,17,4,6,20,5,21,16,100,200] meta(t::Type{RewriterConfig}) = meta(t, ProtoBuf.DEF_REQ, __fnum_RewriterConfig, ProtoBuf.DEF_VAL, true, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES, ProtoBuf.DEF_ONEOFS, ProtoBuf.DEF_ONEOF_NAMES, ProtoBuf.DEF_FIELD_TYPES) export AutoParallelOptions, ScopedAllocatorOptions, RewriterConfig_Toggle, RewriterConfig_NumIterationsType, RewriterConfig_MemOptType, RewriterConfig_CustomGraphOptimizer_ParameterMapEntry, RewriterConfig_CustomGraphOptimizer, RewriterConfig diff --git a/src/protobufs/saved_model_pb.jl b/src/protobufs/saved_model_pb.jl index 5bb6ecdf..f6ec1628 100644 --- a/src/protobufs/saved_model_pb.jl +++ b/src/protobufs/saved_model_pb.jl @@ -1,5 +1,4 @@ # syntax: proto3 -using Compat using ProtoBuf import ProtoBuf.meta diff --git a/src/protobufs/saver_pb.jl b/src/protobufs/saver_pb.jl index b2d93953..9783648f 100644 --- a/src/protobufs/saver_pb.jl +++ b/src/protobufs/saver_pb.jl @@ -1,5 +1,4 @@ # syntax: proto3 -using Compat using ProtoBuf import ProtoBuf.meta diff --git a/src/protobufs/step_stats_pb.jl b/src/protobufs/step_stats_pb.jl index fc9a3ce7..b3ed8625 100644 --- a/src/protobufs/step_stats_pb.jl +++ b/src/protobufs/step_stats_pb.jl @@ -1,5 +1,4 @@ # syntax: proto3 -using Compat using ProtoBuf import ProtoBuf.meta diff --git a/src/protobufs/tensor_bundle_pb.jl b/src/protobufs/tensor_bundle_pb.jl index 66d8f416..fd9e7b60 100644 --- a/src/protobufs/tensor_bundle_pb.jl +++ b/src/protobufs/tensor_bundle_pb.jl @@ -1,5 +1,4 @@ # syntax: proto3 -using Compat using ProtoBuf import ProtoBuf.meta diff --git a/src/protobufs/tensor_description_pb.jl b/src/protobufs/tensor_description_pb.jl index e2b92c15..f9234172 100644 --- a/src/protobufs/tensor_description_pb.jl +++ b/src/protobufs/tensor_description_pb.jl @@ -1,5 +1,4 @@ # syntax: proto3 -using Compat using ProtoBuf import ProtoBuf.meta diff --git a/src/protobufs/tensor_pb.jl b/src/protobufs/tensor_pb.jl index b2a50bac..3fd483b7 100644 --- a/src/protobufs/tensor_pb.jl +++ b/src/protobufs/tensor_pb.jl @@ -1,5 +1,4 @@ # syntax: proto3 -using Compat using ProtoBuf import ProtoBuf.meta diff --git a/src/protobufs/tensor_shape_pb.jl b/src/protobufs/tensor_shape_pb.jl index 47311543..a2ef4fc3 100644 --- a/src/protobufs/tensor_shape_pb.jl +++ b/src/protobufs/tensor_shape_pb.jl @@ -1,5 +1,4 @@ # syntax: proto3 -using Compat using ProtoBuf import ProtoBuf.meta diff --git a/src/protobufs/tensor_slice_pb.jl b/src/protobufs/tensor_slice_pb.jl index e736693e..5bfaec37 100644 --- a/src/protobufs/tensor_slice_pb.jl +++ b/src/protobufs/tensor_slice_pb.jl @@ -1,5 +1,4 @@ # syntax: proto3 -using Compat using ProtoBuf import ProtoBuf.meta diff --git a/src/protobufs/tensorflow.jl b/src/protobufs/tensorflow.jl index bc119365..3b619413 100644 --- a/src/protobufs/tensorflow.jl +++ b/src/protobufs/tensorflow.jl @@ -1,5 +1,4 @@ module tensorflow - using Compat const _ProtoBuf_Top_ = @static isdefined(parentmodule(@__MODULE__), :_ProtoBuf_Top_) ? (parentmodule(@__MODULE__))._ProtoBuf_Top_ : parentmodule(@__MODULE__) include("checkpointable_object_graph_pb.jl") include("cluster_pb.jl") @@ -27,7 +26,6 @@ module tensorflow include("tensorflow_server_pb.jl") include("named_tensor_pb.jl") module error - using Compat const _ProtoBuf_Top_ = @static isdefined(parentmodule(@__MODULE__), :_ProtoBuf_Top_) ? (parentmodule(@__MODULE__))._ProtoBuf_Top_ : parentmodule(@__MODULE__) include("error_codes_pb.jl") end @@ -42,12 +40,10 @@ module tensorflow include("transport_options_pb.jl") include("worker_pb.jl") module eager - using Compat const _ProtoBuf_Top_ = @static isdefined(parentmodule(@__MODULE__), :_ProtoBuf_Top_) ? (parentmodule(@__MODULE__))._ProtoBuf_Top_ : parentmodule(@__MODULE__) include("eager_service_pb.jl") end module grpc - using Compat const _ProtoBuf_Top_ = @static isdefined(parentmodule(@__MODULE__), :_ProtoBuf_Top_) ? (parentmodule(@__MODULE__))._ProtoBuf_Top_ : parentmodule(@__MODULE__) include("master_service_pb.jl") include("worker_service_pb.jl") diff --git a/src/protobufs/tensorflow_server_pb.jl b/src/protobufs/tensorflow_server_pb.jl index 596c4107..07823608 100644 --- a/src/protobufs/tensorflow_server_pb.jl +++ b/src/protobufs/tensorflow_server_pb.jl @@ -1,5 +1,4 @@ # syntax: proto3 -using Compat using ProtoBuf import ProtoBuf.meta diff --git a/src/protobufs/transport_options_pb.jl b/src/protobufs/transport_options_pb.jl index 432b664f..150f5c63 100644 --- a/src/protobufs/transport_options_pb.jl +++ b/src/protobufs/transport_options_pb.jl @@ -1,10 +1,9 @@ # syntax: proto3 -using Compat using ProtoBuf import ProtoBuf.meta mutable struct RecvBufRespExtra <: ProtoType - tensor_content::Array{UInt8,1} + tensor_content::Base.Vector{Array{UInt8,1}} RecvBufRespExtra(; kwargs...) = (o=new(); fillunset(o); isempty(kwargs) || ProtoBuf._protobuild(o, kwargs); o) end #mutable struct RecvBufRespExtra diff --git a/src/protobufs/types_pb.jl b/src/protobufs/types_pb.jl index 921e91c2..5f0da197 100644 --- a/src/protobufs/types_pb.jl +++ b/src/protobufs/types_pb.jl @@ -1,5 +1,4 @@ # syntax: proto3 -using Compat using ProtoBuf import ProtoBuf.meta diff --git a/src/protobufs/versions_pb.jl b/src/protobufs/versions_pb.jl index 1116b4a9..d6e37717 100644 --- a/src/protobufs/versions_pb.jl +++ b/src/protobufs/versions_pb.jl @@ -1,5 +1,4 @@ # syntax: proto3 -using Compat using ProtoBuf import ProtoBuf.meta diff --git a/src/protobufs/worker_pb.jl b/src/protobufs/worker_pb.jl index 64334002..c1573608 100644 --- a/src/protobufs/worker_pb.jl +++ b/src/protobufs/worker_pb.jl @@ -1,5 +1,4 @@ # syntax: proto3 -using Compat using ProtoBuf import ProtoBuf.meta import ProtoBuf.google.protobuf @@ -244,6 +243,7 @@ meta(t::Type{CompleteInstanceRequest}) = meta(t, ProtoBuf.DEF_REQ, ProtoBuf.DEF_ mutable struct CompleteInstanceResponse <: ProtoType instance_key::Int32 source_rank::Int32 + communicator_key::Array{UInt8,1} CompleteInstanceResponse(; kwargs...) = (o=new(); fillunset(o); isempty(kwargs) || ProtoBuf._protobuild(o, kwargs); o) end #mutable struct CompleteInstanceResponse diff --git a/src/protobufs/worker_service_pb.jl b/src/protobufs/worker_service_pb.jl index 5e041449..fd2c45d5 100644 --- a/src/protobufs/worker_service_pb.jl +++ b/src/protobufs/worker_service_pb.jl @@ -1,5 +1,4 @@ # syntax: proto3 -using Compat using ProtoBuf import ProtoBuf.meta import ._ProtoBuf_Top_.tensorflow