Commit 5a6c00d8 authored by Yu Gan's avatar Yu Gan

Initial commit of media microservices

parent dbf98f33
cmake-build-debug-remote-host/*
cmake-build-debug-remote/*
wrk2/*
\ No newline at end of file
cmake_minimum_required(VERSION 3.5)
project(media_microservices)
set(CMAKE_CXX_STANDARD 14)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
#set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -W -Wall -Wextra -O2")
SET(CMAKE_INSTALL_PREFIX /usr/local/bin)
add_subdirectory(src)
#add_subdirectory(test)
#enable_testing()
#add_test(
# testClientPool
# testClientPool
#)
FROM yg397/thrift-microservice-deps:xenial
ARG NUM_CPUS=40
COPY ./ /media-microservices
RUN cd /media-microservices \
&& mkdir -p build \
&& cd build \
&& cmake .. \
&& make -j${NUM_CPUS} \
&& make install
WORKDIR /media-microservices
\ No newline at end of file
# Media Microservices
## Dependencies
- thrift C++ library
- mongo-c-driver
- libmemcached
- nlohmann/json https://nlohmann.github.io/json/
## Pre-requirements
- Docker
- Docker-compose
- Python 3.5+ (with asyncio and aiohttp)
- libssl-dev (apt-get install libssl-dev)
- libz-dev (apt-get install libz-dev)
- luarocks (apt-get install luarocks)
- luasocket (luarocks install luasocket)
## Running the media service application
### Before you start
- Install Docker and Docker Compose.
- Make sure the following ports are available: port `8080` for Nginx frontend and
`16686` for Jaeger.
### Start docker containers
Start docker containers by running `docker-compose up -d`. All images will be
pulled from Docker Hub.
### Register users and movie information
```
python3 scripts/write_movie_info.py && scripts//register_users.sh
```
### Running HTTP workload generator
#### Make
```bash
cd wrk2
make
```
#### Compose reviews
```bash
cd wrk2
./wrk -D exp -t <num-threads> -c <num-conns> -d <duration> -L -s ./scripts/media-microservices/compose-review.lua http://localhost:8080/wrk2-api/review/compose -R <reqs-per-sec>
```
#### View Jaeger traces
View Jaeger traces by accessing `http://localhost:16686`
################################################################################
# HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems®.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
################################################################################
# - Try to find the libmemcached library # Once done this will define #
# LIBMEMCACHED_FOUND - system has the libmemcached library
# LIBMEMCACHED_INCLUDE_DIR - the libmemcached include directory(s)
# LIBMEMCACHED_LIBRARIES - The libraries needed to use libmemcached
# If the memcached libraries are found on the system, we assume they exist natively and dependencies
# can be handled through package management. If the libraries are not found, and if
# MEMCACHED_USE_EXTERNAL_LIBRARY is ON, we will fetch, build, and include a copy of the neccessary
# Libraries.
option(MEMCACHED_USE_EXTERNAL_LIBRARY "Pull and build source from external location if local is not found" ON)
# Search for native library to build against
if(WIN32)
set(libmemcached_lib "libmemcached")
set(libmemcachedUtil_lib "libmemcachedutil")
else()
set(libmemcached_lib "memcached")
set(libmemcachedUtil_lib "memcachedutil")
endif()
find_path(LIBMEMCACHED_INCLUDE_DIR libmemcached/memcached.hpp PATHS /usr/include /usr/share/include /usr/local/include PATH_SUFFIXES libmemcached)
find_library(LIBMEMCACHEDCORE_LIBRARY NAMES ${libmemcached_lib} PATHS /usr/lib usr/lib/libmemcached /usr/share /usr/lib64 /usr/local/lib /usr/local/lib64)
find_library(LIBMEMCACHEDUTIL_LIBRARY NAMES ${libmemcachedUtil_lib} PATHS /usr/lib /usr/share /usr/lib64 /usr/local/lib /usr/local/lib64)
set(LIBMEMCACHED_LIBRARIES ${LIBMEMCACHEDCORE_LIBRARY} ${LIBMEMCACHEDUTIL_LIBRARY})
if(LIBMEMCACHED_INCLUDE_DIR)
if(EXISTS "${LIBMEMCACHED_INCLUDE_DIR}/libmemcached-1.0/configure.h")
file(STRINGS "${LIBMEMCACHED_INCLUDE_DIR}/libmemcached-1.0/configure.h" version REGEX "#define LIBMEMCACHED_VERSION_STRING")
string(REGEX REPLACE "#define LIBMEMCACHED_VERSION_STRING " "" version "${version}")
string(REGEX REPLACE "\"" "" version "${version}")
set(LIBMEMCACHED_VERSION_STRING ${version})
if("${LIBMEMCACHED_VERSION_STRING}" VERSION_EQUAL "${LIBMEMCACHED_FIND_VERSION}" OR "${LIBMEMCACHED_VERSION_STRING}" VERSION_GREATER "${LIBMEMCACHED_FIND_VERSION}")
set(LIBMEMCACHED_VERSION_OK 1)
set(MSG "${DEFAULT_MSG}")
else()
set(LIBMEMCACHED_VERSION_OK 0)
set(MSG "libmemcached version '${LIBMEMCACHED_VERSION_STRING}' incompatible with min version>=${LIBMEMCACHED_FIND_VERSION}")
endif()
endif()
endif()
if((LIBMEMCACHEDCORE_LIBRARY STREQUAL "LIBMEMCACHEDCORE_LIBRARY-NOTFOUND"
OR LIBMEMCACHEDUTIL_LIBRARY STREQUAL "LIBMEMCACHEDUTIL_LIBRARY-NOTFOUND"
OR LIBMEMCACHED_INCLUDE_DIR STREQUAL "LIBMEMCACHED_INCLUDE_DIR-NOTFOUND"
OR NOT LIBMEMCACHED_VERSION_OK)
AND MEMCACHED_USE_EXTERNAL_LIBRARY)
# Currently libmemcached versions are not sufficient on ubuntu 12.04 and 14.04 LTS
# until then, we build the required libraries from source
if(NOT TARGET generate-libmemcached)
set(LIBMEMCACHED_URL https://launchpad.net/libmemcached/1.0/${LIBMEMCACHED_FIND_VERSION}/+download/libmemcached-${LIBMEMCACHED_FIND_VERSION}.tar.gz)
include(ExternalProject)
ExternalProject_Add(
generate-libmemcached
URL ${LIBMEMCACHED_URL}
DOWNLOAD_NO_PROGRESS 1
TIMEOUT 15
DOWNLOAD_DIR ${CMAKE_BINARY_DIR}/downloads
SOURCE_DIR ${CMAKE_BINARY_DIR}/downloads/libmemcached
CONFIGURE_COMMAND "${CMAKE_BINARY_DIR}/downloads/libmemcached/configure" --prefix=${INSTALL_DIR} LDFLAGS=-L${LIB_PATH}
BUILD_COMMAND ${CMAKE_MAKE_PROGRAM} LDFLAGS=-Wl,-rpath-link,${LIB_PATH}
BINARY_DIR ${CMAKE_BINARY_DIR}/build-libmemcached
INSTALL_COMMAND "")
add_library(libmemcached SHARED IMPORTED GLOBAL)
add_library(libmemcachedutil SHARED IMPORTED GLOBAL)
set_property(TARGET libmemcached
PROPERTY IMPORTED_LOCATION ${CMAKE_BINARY_DIR}/build-libmemcached/libmemcached/.libs/libmemcached.so.11.0.0)
set_property(TARGET libmemcachedutil
PROPERTY IMPORTED_LOCATION ${CMAKE_BINARY_DIR}/build-libmemcached/libmemcached/.libs/libmemcachedutil.so.2.0.0)
set_property(TARGET libmemcached
PROPERTY IMPORTED_LINK_DEPENDENT_LIBRARIES libmemcachedutil)
add_dependencies(libmemcached generate-libmemcached)
add_dependencies(libmemcachedutil generate-libmemcached)
if(PLATFORM)
install(CODE "set(ENV{LD_LIBRARY_PATH} \"\$ENV{LD_LIBRARY_PATH}:${CMAKE_BINARY_DIR}:${CMAKE_BINARY_DIR}/build-libmemcached/libmemcached/.libs\")")
install(PROGRAMS
${CMAKE_BINARY_DIR}/build-libmemcached/libmemcached/.libs/libmemcached.so
${CMAKE_BINARY_DIR}/build-libmemcached/libmemcached/.libs/libmemcached.so.11
${CMAKE_BINARY_DIR}/build-libmemcached/libmemcached/.libs/libmemcached.so.11.0.0
${CMAKE_BINARY_DIR}/build-libmemcached/libmemcached/.libs/libmemcachedutil.so
${CMAKE_BINARY_DIR}/build-libmemcached/libmemcached/.libs/libmemcachedutil.so.2
${CMAKE_BINARY_DIR}/build-libmemcached/libmemcached/.libs/libmemcachedutil.so.2.0.0
DESTINATION lib)
endif()
endif()
set(LIBMEMCACHEDCORE_LIBRARY $<TARGET_FILE:libmemcached>)
set(LIBMEMCACHEDUTIL_LIBRARY $<TARGET_FILE:libmemcachedutil>)
set(LIBMEMCACHED_LIBRARIES $<TARGET_FILE:libmemcached> $<TARGET_FILE:libmemcachedutil>)
set(LIBMEMCACHED_INCLUDE_DIR ${CMAKE_BINARY_DIR}/downloads/libmemcached)
# always assumed to be ok
set(LIBMEMCACHED_VERSION_OK 1)
else()
set(MEMCACHED_USE_EXTERNAL_LIBRARY OFF)
endif()
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(
LIBMEMCACHED DEFAULT_MSG
LIBMEMCACHEDCORE_LIBRARY
LIBMEMCACHEDUTIL_LIBRARY
LIBMEMCACHED_INCLUDE_DIR
LIBMEMCACHED_VERSION_OK)
mark_as_advanced(LIBMEMCACHED_INCLUDE_DIR LIBMEMCACHED_LIBRARIES LIBMEMCACHEDCORE_LIBRARY LIBMEMCACHEDUTIL_LIBRARY)
\ No newline at end of file
##############################################################################
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
##############################################################################
# - Find Thrift (a cross platform RPC lib/tool)
# THRIFT_ROOT hints the location
#
# This module defines
# THRIFT_VERSION, version string of ant if found
# THRIFT_INCLUDE_DIR, where to find THRIFT headers
# THRIFT_CONTRIB_DIR, where contrib thrift files (e.g. fb303.thrift) are installed
# THRIFT_LIBS, THRIFT libraries
# thriftstatic - imported static library
# prefer the thrift version supplied in THRIFT_HOME
if (NOT THRIFT_FIND_QUIETLY)
message(STATUS "THRIFT_HOME: $ENV{THRIFT_HOME}")
endif()
find_path(THRIFT_INCLUDE_DIR thrift/Thrift.h HINTS
${THRIFT_ROOT}/include
$ENV{THRIFT_HOME}/include/
/usr/local/include/
/opt/local/include/
)
find_path(THRIFT_CONTRIB_DIR share/fb303/if/fb303.thrift HINTS
${THRIFT_ROOT}/include
$ENV{THRIFT_HOME}
/usr/local/
)
set(THRIFT_LIB_PATHS
${THRIFT_ROOT}/lib
$ENV{THRIFT_HOME}/lib
/usr/local/lib
/opt/local/lib)
find_path(THRIFT_STATIC_LIB_PATH libthrift.a PATHS ${THRIFT_LIB_PATHS})
# prefer the thrift version supplied in THRIFT_HOME
find_library(THRIFT_LIB NAMES thrift HINTS ${THRIFT_LIB_PATHS})
find_program(THRIFT_COMPILER thrift
${THRIFT_ROOT}/bin
$ENV{THRIFT_HOME}/bin
/usr/local/bin
/usr/bin
NO_DEFAULT_PATH
)
if (THRIFT_LIB)
set(THRIFT_LIBS ${THRIFT_LIB})
set(THRIFT_STATIC_LIB ${THRIFT_STATIC_LIB_PATH}/libthrift.a)
exec_program(${THRIFT_COMPILER}
ARGS -version OUTPUT_VARIABLE THRIFT_VERSION RETURN_VALUE THRIFT_RETURN)
if (NOT THRIFT_FIND_QUIETLY)
message(STATUS "Thrift version: ${THRIFT_VERSION}")
endif ()
set(THRIFT_FOUND TRUE)
# for static linking with Thrift, THRIFT_STATIC_LIB is set in FindThrift.cmake
add_library(thriftstatic STATIC IMPORTED)
set_target_properties(thriftstatic PROPERTIES IMPORTED_LOCATION ${THRIFT_STATIC_LIB})
else ()
set(THRIFT_FOUND FALSE)
message(FATAL_ERROR "Thrift compiler/libraries NOT found. "
"Thrift support will be disabled (${THRIFT_RETURN}, "
"${THRIFT_INCLUDE_DIR}, ${THRIFT_LIB})")
endif ()
mark_as_advanced(
THRIFT_LIB
THRIFT_COMPILER
THRIFT_INCLUDE_DIR
thriftstatic
)
\ No newline at end of file
disabled: false
reporter:
logSpans: false
localAgentHostPort: "jaeger:6831"
sampler:
type: const
param: 1
\ No newline at end of file
rs.initiate(
{
"_id": "movie-id-mongodb-config",
"configsvr": "true",
"version": 1,
"members": [
{
"_id": "movie-id-mongodb-config",
"host": "movie-id-mongodb-config:27017"
}
]
}
)
\ No newline at end of file
sh.addShard("movie-id-mongodb-shard-0/movie-id-mongodb-shard-0_0:27017")
sh.addShard("movie-id-mongodb-shard-1/movie-id-mongodb-shard-1_0:27017")
sh.addShard("movie-id-mongodb-shard-2/movie-id-mongodb-shard-2_0:27017")
rs.initiate(
{
"_id": "movie-id-mongodb-shard-0",
"version": 1,
"members": [
{
"_id": "movie-id-mongodb-shard-0_0",
"host": "movie-id-mongodb-shard-0_0:27017"
}
]
}
)
\ No newline at end of file