protobuf produces non-compilable C++ code in simple case

[smgutstein]

What version of protobuf and what language are you using?
Version: master/v3.6.0/v3.5.0 etc.
Language: C++/Java/Python/C#/Ruby/PHP/Objective-C/Javascript

I am using version 3.11.2 of protobuf with C++:

$ protoc --version
libprotoc 3.11.2

What runtime / compiler are you using (e.g., python version or gcc version)
The version of g++ I'm using is 7.4.0:

$ g++ --version
g++ (Ubuntu 7.4.0-1ubuntu1~18.04.1) 7.4.0

What operating system (Linux, Windows, ...) and version?
My OS is Ubuntu 18.04:

$ lsb_release -a
No LSB modules are available.
Distributor ID:	Ubuntu
Description:	Ubuntu 18.04.3 LTS
Release:	18.04
Codename:	bionic

What did you do?

I created the following simple example of protobuf code:

syntax = "proto2";

package test1;

message Datum {
  required int32 channels = 1;
}

However, when I compiled the protobuf code in C++, the C++ code wouldn't compile.

$ protoc --cpp_out=. temp5.proto
$ g++ temp5.pb.cc 
temp5.pb.cc: In member function ‘virtual const char* test1::Datum::_InternalParse(const char*, google::protobuf::internal::ParseContext*)’:
temp5.pb.cc:150:58: error: ‘ReadVarint’ is not a member of ‘google::protobuf::internal’
           channels_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr);
                                                          ^~~~~~~~~~
temp5.pb.cc:150:58: note: suggested alternative: ‘ReadVarint32’
           channels_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr);
                                                          ^~~~~~~~~~
                                                          ReadVarint32

The code that was produced (i.e. the non-compilable code) was:

// Generated by the protocol buffer compiler.  DO NOT EDIT!
// source: temp5.proto

#include "temp5.pb.h"

#include <algorithm>

#include <google/protobuf/io/coded_stream.h>
#include <google/protobuf/extension_set.h>
#include <google/protobuf/wire_format_lite.h>
#include <google/protobuf/descriptor.h>
#include <google/protobuf/generated_message_reflection.h>
#include <google/protobuf/reflection_ops.h>
#include <google/protobuf/wire_format.h>
// @@protoc_insertion_point(includes)
#include <google/protobuf/port_def.inc>
namespace test1 {
class DatumDefaultTypeInternal {
 public:
  ::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed<Datum> _instance;
} _Datum_default_instance_;
}  // namespace test1
static void InitDefaultsscc_info_Datum_temp5_2eproto() {
  GOOGLE_PROTOBUF_VERIFY_VERSION;

  {
    void* ptr = &::test1::_Datum_default_instance_;
    new (ptr) ::test1::Datum();
    ::PROTOBUF_NAMESPACE_ID::internal::OnShutdownDestroyMessage(ptr);
  }
  ::test1::Datum::InitAsDefaultInstance();
}

::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_Datum_temp5_2eproto =
    {{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 0, 0, InitDefaultsscc_info_Datum_temp5_2eproto}, {}};

static ::PROTOBUF_NAMESPACE_ID::Metadata file_level_metadata_temp5_2eproto[1];
static constexpr ::PROTOBUF_NAMESPACE_ID::EnumDescriptor const** file_level_enum_descriptors_temp5_2eproto = nullptr;
static constexpr ::PROTOBUF_NAMESPACE_ID::ServiceDescriptor const** file_level_service_descriptors_temp5_2eproto = nullptr;

const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_temp5_2eproto::offsets[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) = {
  PROTOBUF_FIELD_OFFSET(::test1::Datum, _has_bits_),
  PROTOBUF_FIELD_OFFSET(::test1::Datum, _internal_metadata_),
  ~0u,  // no _extensions_
  ~0u,  // no _oneof_case_
  ~0u,  // no _weak_field_map_
  PROTOBUF_FIELD_OFFSET(::test1::Datum, channels_),
  0,
};
static const ::PROTOBUF_NAMESPACE_ID::internal::MigrationSchema schemas[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) = {
  { 0, 6, sizeof(::test1::Datum)},
};

static ::PROTOBUF_NAMESPACE_ID::Message const * const file_default_instances[] = {
  reinterpret_cast<const ::PROTOBUF_NAMESPACE_ID::Message*>(&::test1::_Datum_default_instance_),
};

const char descriptor_table_protodef_temp5_2eproto[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) =
  "\n\013temp5.proto\022\005test1\"\031\n\005Datum\022\020\n\010channel"
  "s\030\001 \001(\005"
  ;
static const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable*const descriptor_table_temp5_2eproto_deps[1] = {
};
static ::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase*const descriptor_table_temp5_2eproto_sccs[1] = {
  &scc_info_Datum_temp5_2eproto.base,
};
static ::PROTOBUF_NAMESPACE_ID::internal::once_flag descriptor_table_temp5_2eproto_once;
static bool descriptor_table_temp5_2eproto_initialized = false;
const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable descriptor_table_temp5_2eproto = {
  &descriptor_table_temp5_2eproto_initialized, descriptor_table_protodef_temp5_2eproto, "temp5.proto", 47,
  &descriptor_table_temp5_2eproto_once, descriptor_table_temp5_2eproto_sccs, descriptor_table_temp5_2eproto_deps, 1, 0,
  schemas, file_default_instances, TableStruct_temp5_2eproto::offsets,
  file_level_metadata_temp5_2eproto, 1, file_level_enum_descriptors_temp5_2eproto, file_level_service_descriptors_temp5_2eproto,
};

// Force running AddDescriptors() at dynamic initialization time.
static bool dynamic_init_dummy_temp5_2eproto = (  ::PROTOBUF_NAMESPACE_ID::internal::AddDescriptors(&descriptor_table_temp5_2eproto), true);
namespace test1 {

// ===================================================================

void Datum::InitAsDefaultInstance() {
}
class Datum::_Internal {
 public:
  using HasBits = decltype(std::declval<Datum>()._has_bits_);
  static void set_has_channels(HasBits* has_bits) {
    (*has_bits)[0] |= 1u;
  }
};

Datum::Datum()
  : ::PROTOBUF_NAMESPACE_ID::Message(), _internal_metadata_(nullptr) {
  SharedCtor();
  // @@protoc_insertion_point(constructor:test1.Datum)
}
Datum::Datum(const Datum& from)
  : ::PROTOBUF_NAMESPACE_ID::Message(),
      _internal_metadata_(nullptr),
      _has_bits_(from._has_bits_) {
  _internal_metadata_.MergeFrom(from._internal_metadata_);
  channels_ = from.channels_;
  // @@protoc_insertion_point(copy_constructor:test1.Datum)
}

void Datum::SharedCtor() {
  channels_ = 0;
}

Datum::~Datum() {
  // @@protoc_insertion_point(destructor:test1.Datum)
  SharedDtor();
}

void Datum::SharedDtor() {
}

void Datum::SetCachedSize(int size) const {
  _cached_size_.Set(size);
}
const Datum& Datum::default_instance() {
  ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&::scc_info_Datum_temp5_2eproto.base);
  return *internal_default_instance();
}


void Datum::Clear() {
// @@protoc_insertion_point(message_clear_start:test1.Datum)
  ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0;
  // Prevent compiler warnings about cached_has_bits being unused
  (void) cached_has_bits;

  channels_ = 0;
  _has_bits_.Clear();
  _internal_metadata_.Clear();
}

const char* Datum::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) {
#define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure
  _Internal::HasBits has_bits{};
  while (!ctx->Done(&ptr)) {
    ::PROTOBUF_NAMESPACE_ID::uint32 tag;
    ptr = ::PROTOBUF_NAMESPACE_ID::internal::ReadTag(ptr, &tag);
    CHK_(ptr);
    switch (tag >> 3) {
      // optional int32 channels = 1;
      case 1:
        if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 8)) {
          _Internal::set_has_channels(&has_bits);
          channels_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr);
          CHK_(ptr);
        } else goto handle_unusual;
        continue;
      default: {
      handle_unusual:
        if ((tag & 7) == 4 || tag == 0) {
          ctx->SetLastTag(tag);
          goto success;
        }
        ptr = UnknownFieldParse(tag, &_internal_metadata_, ptr, ctx);
        CHK_(ptr != nullptr);
        continue;
      }
    }  // switch
  }  // while
success:
  _has_bits_.Or(has_bits);
  return ptr;
failure:
  ptr = nullptr;
  goto success;
#undef CHK_
}

::PROTOBUF_NAMESPACE_ID::uint8* Datum::_InternalSerialize(
    ::PROTOBUF_NAMESPACE_ID::uint8* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const {
  // @@protoc_insertion_point(serialize_to_array_start:test1.Datum)
  ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0;
  (void) cached_has_bits;

  cached_has_bits = _has_bits_[0];
  // optional int32 channels = 1;
  if (cached_has_bits & 0x00000001u) {
    target = stream->EnsureSpace(target);
    target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(1, this->_internal_channels(), target);
  }

  if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) {
    target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::InternalSerializeUnknownFieldsToArray(
        _internal_metadata_.unknown_fields(), target, stream);
  }
  // @@protoc_insertion_point(serialize_to_array_end:test1.Datum)
  return target;
}

size_t Datum::ByteSizeLong() const {
// @@protoc_insertion_point(message_byte_size_start:test1.Datum)
  size_t total_size = 0;

  ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0;
  // Prevent compiler warnings about cached_has_bits being unused
  (void) cached_has_bits;

  // optional int32 channels = 1;
  cached_has_bits = _has_bits_[0];
  if (cached_has_bits & 0x00000001u) {
    total_size += 1 +
      ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size(
        this->_internal_channels());
  }

  if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) {
    return ::PROTOBUF_NAMESPACE_ID::internal::ComputeUnknownFieldsSize(
        _internal_metadata_, total_size, &_cached_size_);
  }
  int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size);
  SetCachedSize(cached_size);
  return total_size;
}

void Datum::MergeFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) {
// @@protoc_insertion_point(generalized_merge_from_start:test1.Datum)
  GOOGLE_DCHECK_NE(&from, this);
  const Datum* source =
      ::PROTOBUF_NAMESPACE_ID::DynamicCastToGenerated<Datum>(
          &from);
  if (source == nullptr) {
  // @@protoc_insertion_point(generalized_merge_from_cast_fail:test1.Datum)
    ::PROTOBUF_NAMESPACE_ID::internal::ReflectionOps::Merge(from, this);
  } else {
  // @@protoc_insertion_point(generalized_merge_from_cast_success:test1.Datum)
    MergeFrom(*source);
  }
}

void Datum::MergeFrom(const Datum& from) {
// @@protoc_insertion_point(class_specific_merge_from_start:test1.Datum)
  GOOGLE_DCHECK_NE(&from, this);
  _internal_metadata_.MergeFrom(from._internal_metadata_);
  ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0;
  (void) cached_has_bits;

  if (from._internal_has_channels()) {
    _internal_set_channels(from._internal_channels());
  }
}

void Datum::CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) {
// @@protoc_insertion_point(generalized_copy_from_start:test1.Datum)
  if (&from == this) return;
  Clear();
  MergeFrom(from);
}

void Datum::CopyFrom(const Datum& from) {
// @@protoc_insertion_point(class_specific_copy_from_start:test1.Datum)
  if (&from == this) return;
  Clear();
  MergeFrom(from);
}

bool Datum::IsInitialized() const {
  return true;
}

void Datum::InternalSwap(Datum* other) {
  using std::swap;
  _internal_metadata_.Swap(&other->_internal_metadata_);
  swap(_has_bits_[0], other->_has_bits_[0]);
  swap(channels_, other->channels_);
}

::PROTOBUF_NAMESPACE_ID::Metadata Datum::GetMetadata() const {
  return GetMetadataStatic();
}


// @@protoc_insertion_point(namespace_scope)
}  // namespace test1
PROTOBUF_NAMESPACE_OPEN
template<> PROTOBUF_NOINLINE ::test1::Datum* Arena::CreateMaybeMessage< ::test1::Datum >(Arena* arena) {
  return Arena::CreateInternal< ::test1::Datum >(arena);
}
PROTOBUF_NAMESPACE_CLOSE

// @@protoc_insertion_point(global_scope)
#include <google/protobuf/port_undef.inc>

-->

I created this sample, because I was trying to bulid Caffe. Its make process failed wth the same message, and this protobuf code was my attempt to recreate the error I was seeing in the simplest way possible.

[rafi-kamal]

Possible duplicate of #7107

[smgutstein]

The problem (for me) seems to lie in a bad version of libprotobuf that is installed by conda. Consider the simple proto code I created above (i.e. temp5.proto). If I create a fresh conda` environment and try to compile this code, it works.

The version of libprotobuf used is the one I installed by using git clone and following the installation instructions. It was commit 6f9d48814928d99c079ac2dcbd4c2a50d6e97915 . To see it work, look at my output below:

(base) me@balin:~$ cd Projects/proto_temp/

## Create a fresh conda environment - 
## because it does not have a copy of libprotobuf, it will use the one I installed by git cloning...

(base) me@balin:~/Projects/proto_temp$ conda create --name test_env python=3.7
Collecting package metadata (current_repodata.json): done
Solving environment: done

## Package Plan ##

  environment location: /home/me/anaconda2/envs/test_env

  added / updated specs:
    - python=3.7


The following packages will be downloaded:

    package                    |            build
    ---------------------------|-----------------
    ca-certificates-2020.1.1   |                0         125 KB
    pip-20.0.2                 |           py37_1         1.7 MB
    wheel-0.34.1               |           py37_0          50 KB
    ------------------------------------------------------------
                                           Total:         1.8 MB

The following NEW packages will be INSTALLED:

  _libgcc_mutex      pkgs/main/linux-64::_libgcc_mutex-0.1-main
  ca-certificates    pkgs/main/linux-64::ca-certificates-2020.1.1-0
  certifi            pkgs/main/linux-64::certifi-2019.11.28-py37_0
  ld_impl_linux-64   pkgs/main/linux-64::ld_impl_linux-64-2.33.1-h53a641e_7
  libedit            pkgs/main/linux-64::libedit-3.1.20181209-hc058e9b_0
  libffi             pkgs/main/linux-64::libffi-3.2.1-hd88cf55_4
  libgcc-ng          pkgs/main/linux-64::libgcc-ng-9.1.0-hdf63c60_0
  libstdcxx-ng       pkgs/main/linux-64::libstdcxx-ng-9.1.0-hdf63c60_0
  ncurses            pkgs/main/linux-64::ncurses-6.1-he6710b0_1
  openssl            pkgs/main/linux-64::openssl-1.1.1d-h7b6447c_3
  pip                pkgs/main/linux-64::pip-20.0.2-py37_1
  python             pkgs/main/linux-64::python-3.7.6-h0371630_2
  readline           pkgs/main/linux-64::readline-7.0-h7b6447c_5
  setuptools         pkgs/main/linux-64::setuptools-45.1.0-py37_0
  sqlite             pkgs/main/linux-64::sqlite-3.30.1-h7b6447c_0
  tk                 pkgs/main/linux-64::tk-8.6.8-hbc83047_0
  wheel              pkgs/main/linux-64::wheel-0.34.1-py37_0
  xz                 pkgs/main/linux-64::xz-5.2.4-h14c3975_4
  zlib               pkgs/main/linux-64::zlib-1.2.11-h7b6447c_3


Proceed ([y]/n)? y


Downloading and Extracting Packages
pip-20.0.2           | 1.7 MB    | ##################################################################################################################################################### | 100% 
ca-certificates-2020 | 125 KB    | ##################################################################################################################################################### | 100% 
wheel-0.34.1         | 50 KB     | ##################################################################################################################################################### | 100% 
Preparing transaction: done
Verifying transaction: done
Executing transaction: done
#
# To activate this environment, use
#
#     $ conda activate test_env
#
# To deactivate an active environment, use
#
#     $ conda deactivate

## Activate fresh environment
(base) me@balin:~/Projects/proto_temp$ conda activate test_env

## Successfully compile temp5.proto
(test_env) me@balin:~/Projects/proto_temp$ protoc --cpp_out=. temp5.proto
(test_env) me@balin:~/Projects/proto_temp$ g++ temp5.pb.cc -c`
(test_env) me@balin

Next, if I conda install protobuf, I will recreate the original error:

(test_env) me@balin:~/Projects/proto_temp$ conda install -c anaconda protobuf 
Collecting package metadata (current_repodata.json): done
Solving environment: done

## Package Plan ##

  environment location: /home/me/anaconda2/envs/test_env

  added / updated specs:
    - protobuf


The following packages will be downloaded:

    package                    |            build
    ---------------------------|-----------------
    ca-certificates-2019.11.27 |                0         132 KB  anaconda
    certifi-2019.11.28         |           py37_0         156 KB  anaconda
    six-1.14.0                 |           py37_0          27 KB  anaconda
    ------------------------------------------------------------
                                           Total:         315 KB

The following NEW packages will be INSTALLED:

  libprotobuf        anaconda/linux-64::libprotobuf-3.11.2-hd408876_0
  protobuf           anaconda/linux-64::protobuf-3.11.2-py37he6710b0_0
  six                anaconda/linux-64::six-1.14.0-py37_0

The following packages will be UPDATED:

  openssl              pkgs/main::openssl-1.1.1d-h7b6447c_3 --> anaconda::openssl-1.1.1-h7b6447c_0

The following packages will be SUPERSEDED by a higher-priority channel:

  ca-certificates     pkgs/main::ca-certificates-2020.1.1-0 --> anaconda::ca-certificates-2019.11.27-0
  certifi                                         pkgs/main --> anaconda


Proceed ([y]/n)? y


Downloading and Extracting Packages
six-1.14.0           | 27 KB     | ##################################################################################################################################################### | 100% 
ca-certificates-2019 | 132 KB    | ##################################################################################################################################################### | 100% 
certifi-2019.11.28   | 156 KB    | ##################################################################################################################################################### | 100% 
Preparing transaction: done
Verifying transaction: done
Executing transaction: done

## Now will get original protobuf error - Note: The version of libprotobuf used is 3.11.2  ##
(test_env) me@balin:~/Projects/proto_temp$ protoc --cpp_out=. temp5.proto
(test_env) me@balin:~/Projects/proto_temp$ g++ temp5.pb.cc -c
temp5.pb.cc: In member function ‘virtual const char* test1::Datum::_InternalParse(const char*, google::protobuf::internal::ParseContext*)’:
temp5.pb.cc:150:58: error: ‘ReadVarint’ is not a member of ‘google::protobuf::internal’
           channels_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr);
                                                          ^~~~~~~~~~
temp5.pb.cc:150:58: note: suggested alternative: ‘ReadVarint32’
           channels_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr);
                                                          ^~~~~~~~~~
                                                          ReadVarint32
(test_env) me@balin:~/Projects/proto_temp$ 

I don't know to whom I should reach out in order to fix the protobuf package installed from the anaconda channel.

[rafi-kamal]

My guess is the protoc you used to compile temp5.proto into was built from the master branch, but libprotobuf uses 3.11.2 which is currently behind the master branch.

Can you try using a protoc built from the 3.11.x branch to compile temp5.proto?

[smgutstein]

The 3.11.x branch seems to successfully compile temp5.proto also, unless I've made a silly mistake...

I performed this fresh install...

$ git clone https://github.com/protocolbuffers/protobuf.git
Cloning into 'protobuf'...
remote: Enumerating objects: 24, done.
remote: Counting objects: 100% (24/24), done.
remote: Compressing objects: 100% (22/22), done.
remote: Total 69568 (delta 6), reused 0 (delta 0), pack-reused 69544
Receiving objects: 100% (69568/69568), 57.95 MiB | 5.69 MiB/s, done.
Resolving deltas: 100% (47597/47597), done.
me@gimli:~/Projects$ cd protobuf/
me@gimli:~/Projects/protobuf$ git status
On branch master
Your branch is up to date with 'origin/master'.

nothing to commit, working tree clean
me@gimli:~/Projects/protobuf$ git checkout 3.11.x
Branch '3.11.x' set up to track remote branch '3.11.x' from 'origin'.
Switched to a new branch '3.11.x'
me@gimli:~/Projects/protobuf$ git branch -v
* 3.11.x 5f3d6599f Update CHANGES.txt for 3.11.3 release (#7162)
  master 4cf5bfee9 Install RubyGem bundler version specified in Gemfile.lock to fix release failure (#7144)

Then followed the make make check sudo make install ldconfig instructions, and successfully compiled temp5.proto

$ protoc --cpp_out=. temp5.proto
me@gimli:~/Projects$ g++ temp5.pb.cc -c
me@gimli:~/Projects$ protoc --version
libprotoc 3.11.3

Hmmm.... there's something I'm missing here. The master branch uses 3.11.2,

me@balin:~/Projects$ protoc --version
libprotoc 3.11.2

but branch 3.11.2 uses libprotoc 3.11.3. I thought you wanted me to try with something less recent than the master branch, which works for me. Were you asking me to make a conda package with branch 3.11.x?

[smgutstein]

Duplicate #7103

We used same sample proto code to generate the same error.

Only difference seems to be #7103 rolled back to last working version of libprotobuf (3.9.2). Here, the most recent version of current version (?) (3.11.2) on the master branch of git repo was used to get correct response. Seems odd that other released versions of 3.11.2 don't work.

[gerben-s]

This issue is fixed in newest versions. Specializing ReadVarint for int32