Added changes to allow specifying the configuration file path

README.md

@@ -206,3 +206,6 @@ On each data node (node, node4):
 </details>
 
 
+
+lustre path in Broadwell
+/lustre/scratch/javier.garciablas

app/server.c

@@ -16,7 +16,7 @@
 #include "cfg_parse.h"
 #include <inttypes.h>
 #include <unistd.h>
-//#include <disk.h>
+// #include <disk.h>
 
 // Pointer to the tree's root node.
 extern GNode *tree_root;
@@ -80,7 +80,7 @@ int32_t main(int32_t argc, char **argv)
 	ucp_ep_h client_ep;
 
 	ucp_am_handler_param_t param;
-	int ret;
+	int ret = 0;
 	ucp_config_t *config;
 	ucp_worker_address_attr_t attr;
 
@@ -91,66 +91,55 @@ int32_t main(int32_t argc, char **argv)
 
 	char tmp_file_path[100];
 
-	t = clock();
+	char *conf_path;
+	char abs_exe_path[1024];
+	char *aux;
 
-	cfg = cfg_init();
+	t = clock();
 
 	/***************************************************************/
 	/******************* PARSE FILE ARGUMENTS **********************/
 	/***************************************************************/
 
-	char path_save[PATH_MAX];
-	char abs_exe_path[PATH_MAX];
-	char abs_exe_path2[PATH_MAX];
-	char *p;
-	char *aux;
-
 	args.type = argv[1][0];
 	args.id = atoi(argv[2]);
-
 	sprintf(tmp_file_path, "/tmp/%c-hercules-%d", args.type, args.id);
 
-	fprintf(stderr, "tmp_file_path=%s\n", tmp_file_path);
-
-	// if (remove(tmp_file_path) == 0) {
-	//     fprintf(stderr, "The file %s is deleted successfully.\n", tmp_file_path);
-	// } else {
-	//     fprintf(stderr, "The file %s is not deleted.\n", tmp_file_path);
-	// }
-
-	if (!(p = strrchr(argv[0], '/')))
-		getcwd(abs_exe_path, sizeof(abs_exe_path));
+	cfg = cfg_init();
+	conf_path = getenv("HERCULES_CONF");
+	if (conf_path != NULL)
+	{
+		ret = cfg_load(cfg, conf_path);
+	}
 	else
 	{
-		*p = '\0';
-		getcwd(path_save, sizeof(path_save));
-		chdir(argv[0]);
-		getcwd(abs_exe_path, sizeof(abs_exe_path));
-		chdir(path_save);
+		ret = 1;
 	}
 
-	strcat(abs_exe_path, "/../conf/hercules.conf");
-	strcpy(abs_exe_path2, abs_exe_path);
-	strcat(abs_exe_path2, "hercules.conf");
-
-	fprintf(stderr, "Trying to load /etc/hercules.conf\n");
-	if (cfg_load(cfg, "/etc/hercules.conf") > 0)
+	if (ret)
 	{
-		fprintf(stderr, "Trying to load %s\n", abs_exe_path);
-		if (cfg_load(cfg, abs_exe_path) > 0)
+		conf_path = (char *)malloc(sizeof(char) * PATH_MAX);
+		strcpy(conf_path, "/etc/hercules.conf");
+		if (cfg_load(cfg, conf_path) > 0)
 		{
-			fprintf(stderr, "Trying to load %s\n", abs_exe_path2);
-			if (cfg_load(cfg, abs_exe_path2) > 0)
+			if (getcwd(abs_exe_path, sizeof(abs_exe_path)) != NULL)
+			{
+				sprintf(conf_path, "%s/%s", abs_exe_path, "../conf/hercules.conf");
+			}
+			else
+			{
+				sprintf(conf_path, "%s", "./hercules.conf");
+			}
+			if (cfg_load(cfg, conf_path) > 0)
 			{
-				fprintf(stderr, "Trying to load ./hercules.conf");
-				if (cfg_load(cfg, "hercules.conf") > 0)
+				sprintf(conf_path, "%s", "./hercules.conf");
+				if (cfg_load(cfg, conf_path) > 0)
 				{
-					// get arguments.
-					fprintf(stderr, "Trying get params");
-					parse_args(argc, argv, &args);
+					cfg_load(cfg, "hercules.conf");
 				}
 			}
 		}
+		free(conf_path);
 	}
 
 	if (cfg_get(cfg, "URI"))
@@ -256,7 +245,7 @@ int32_t main(int32_t argc, char **argv)
 	sprintf(log_path, "./%c-server.%02d-%02d-%02d", args.type, tm.tm_hour, tm.tm_min, tm.tm_sec);
 	// sprintf(log_path, "./%c-server", args.type);
 	slog_init(log_path, IMSS_DEBUG_LEVEL, IMSS_DEBUG_FILE, IMSS_DEBUG_SCREEN, 1, 1, 1, args.id);
-	fprintf(stderr, "IMSS DEBUG FILE AT %s\n", log_path);
+	// fprintf(stderr, "IMSS DEBUG FILE AT %s\n", log_path);
 	slog_info(",Time(msec), Comment, RetCode");
 
 	slog_debug("[SERVER] Starting server.");
@@ -302,7 +291,8 @@ int32_t main(int32_t argc, char **argv)
 		max_storage_size = max_system_ram_allowed;
 	}
 
-	fprintf(stderr, "max_storage_size=%ld\n", max_storage_size);
+	//slog_info("[Server-%c] tmp_file_path=%s, Configuration file loaded %s, max_storage_size=%ld\n", args.type, tmp_file_path, max_storage_size);
+	// fprintf(stderr, "max_storage_size=%ld\n", max_storage_size);
 
 	// init memory pool
 	mem_pool = StsQueue.create();
@@ -637,7 +627,7 @@ int32_t main(int32_t argc, char **argv)
 		if (!args.id)
 			printf("ServerID,'Deployment time (s)'\n");
 
-		fprintf(stderr, "tmp_file_path=%s\n", tmp_file_path);
+		// fprintf(stderr, "tmp_file_path=%s\n", tmp_file_path);
 
 		FILE *tmp_file = tmpfile(); // make the file pointer as temporary file.
 		tmp_file = fopen(tmp_file_path, "w");

bash/c_bash.sh

 #!/bin/bash
 
 ## Uncomment when working in Tucan.
-IOR_PATH=/home/software/io500/bin
-module unload mpi
-module load mpi/mpich3/3.2.1
-
-## Uncomment when working in Unito.
 # IOR_PATH=/home/software/io500/bin
 # module unload mpi
 # module load mpi/mpich3/3.2.1
-# module load mpi/openmpi
+
+## Uncomment when working in Unito.
+IOR_PATH=/beegfs/home/javier.garciablas/io500/bin
+ spack load \
+    cmake@3.24.3%gcc@9.4.0 arch=linux-ubuntu20.04-broadwell \
+    glib@2.74.1%gcc@9.4.0 arch=linux-ubuntu20.04-broadwell \
+    ucx@1.14.0%gcc@9.4.0 arch=linux-ubuntu20.04-broadwell \
+    pcre@8.45%gcc@9.4.0 arch=linux-ubuntu20.04-broadwell \
+    jemalloc
+ spack load openmpi@4.1.5%gcc@9.4.0 arch=linux-ubuntu20.04-broadwell
+
 
 source ./hercules start -m meta_hostfile -d data_hostfile -o /mnt/imss/data.out -s 0
 

bash/c_slurm.sh

 #!/bin/bash
-#SBATCH --job-name=imss    # Job name
-#SBATCH --time=00:60:00               # Time limit hrs:min:sec
-#SBATCH --output=logs/hercules/%j_imss.log   # Standard output and error log
+#SBATCH --job-name=hercules    # Job name
+#SBATCH --time=00:30:00               # Time limit hrs:min:sec
+#SBATCH --output=logs/hercules/%j_hercules.log   # Standard output and error log
 #SBATCH --mem=0
 #SBATCH --overcommit
 #SBATCH --oversubscribe
-#SBATCH --exclusive=user
+##SBATCH --nodelist=broadwell-[038-043]
+##SBATCH --nodelist=broadwell-[000-004]
+#SBATCH --exclude=broadwell-[036-067]
+###SBATCH --exclusive=user
+
+
+CONFIG_PATH=$1
 
 ## Uncomment when working in Tucan.
 # IOR_PATH=/home/software/io500/bin
@@ -19,8 +25,9 @@
     glib@2.74.1%gcc@9.4.0 arch=linux-ubuntu20.04-broadwell \
     ucx@1.14.0%gcc@9.4.0 arch=linux-ubuntu20.04-broadwell \
     pcre@8.45%gcc@9.4.0 arch=linux-ubuntu20.04-broadwell \
-    jemalloc
- spack load openmpi@4.1.5%gcc@9.4.0 arch=linux-ubuntu20.04-broadwell
+    openmpi@4.1.5%gcc@9.4.0 arch=linux-ubuntu20.04-broadwell \
+    jemalloc 
+#  spack load openmpi@4.1.5%gcc@9.4.0 arch=linux-ubuntu20.04-broadwell
 
 ## Uncomment when working in MN4.
 # IOR_PATH=/apps/IOR/3.3.0/INTEL/IMPI/bin
@@ -38,7 +45,12 @@
 
 # start=`date +%s`
 start_=`date +%s.%N`
-source hercules start
+if [ -z "$CONFIG_PATH" ]; then
+   echo "here"
+   source hercules start
+else
+   source hercules start -f "$CONFIG_PATH"
+fi
 end_=`date +%s.%N`
 runtime=$( echo "$end_ - $start_" | bc -l )
 # runtime=$(time source hercules start)
@@ -50,20 +62,22 @@ echo "Hercules started in $runtime seconds, start=$start_, end=$end_"
 
 
 echo "Running clients"
-COMMAND="$IOR_PATH/ior -t 1M -b 10M -s 1 -i 5 -o /mnt/imss/data.out"
+COMMAND="$IOR_PATH/ior -t 1M -b 10M -s 1 -i 5 -F -o /mnt/imss/data.out"
 # COMMAND="./exe_WRITE-AND-READ-TEST-BIFURCADO /mnt/imss/data.out 10240"
 # COMMAND="hostname"
 #COMMAND="echo 'hello' > /tmp/hello"
 #COMMAND="free -h
 
 # set -x
-mpiexec $H_MPI_HOSTFILE_DEF ./client_hostfile -np $H_NCPN \
+echo "mpiexec $H_MPI_HOSTFILE_DEF ./client_hostfile -n $H_NNFC $H_MPI_PPN $H_NCPN \
+	$H_MPI_ENV_DEF $H_POSIX_PRELOAD \
+   	$H_MPI_ENV_DEF IMSS_CONF=$CONFIG_PATH \
+	$COMMAND"
+
+mpiexec $H_MPI_HOSTFILE_DEF ./client_hostfile -n $H_NNFC $H_MPI_PPN $H_NCPN \
 	$H_MPI_ENV_DEF $H_POSIX_PRELOAD \
+   	$H_MPI_ENV_DEF IMSS_CONF=$CONFIG_PATH \
 	$COMMAND
 
-# mpiexec --hostfile ./client_hostfile -npernode $H_NUM_CLIENT \
-# 	-x LD_PRELOAD=$HERCULES_PATH/build/tools/libhercules_posix.so \
-# 	-x IMSS_DEBUG=none \
-#         $COMMAND
 
 ./hercules stop

bash/check-servers.sh

@@ -2,7 +2,7 @@
 
 SERVER_TYPE=$1
 SERVER_NUMBER=$2
-ATTEMPS=10
+ATTEMPS=100
 i=1
 
 FILE="/tmp/$SERVER_TYPE-hercules-$SERVER_NUMBER"

bash/hercules

@@ -20,7 +20,7 @@ WaitForServers() {
     SERVER_TYPE=$1
     shift
     hosts=("$@")
-    echo "hostnames=$hosts"
+    echo "hostnames=${hosts[@]}"
     for node in "${hosts[@]}"
     do
         COMMAND="$H_BASH_PATH/check-servers.sh $SERVER_TYPE $i"
@@ -36,8 +36,8 @@ WaitForServers() {
             StopServers $SERVER_NAME $hosts
             exit 1
         fi
-        echo "[OK] $SERVER_NAME server running in $node"
-        i=$(($i+1))
+        echo "[OK] $SERVER_NAME $i server running in $node"
+        i=$((i+1))
     done
 }
 
@@ -49,7 +49,7 @@ if [[ $STATUS != "stop" && $STATUS != "start" ]]; then
 fi
 
 ## Check if user pass arguments.
-while getopts :m:d:o:c:s: flag
+while getopts :m:d:o:c:s:f: flag
 do
     echo "Entra en getopts"
     case "${flag}" in
@@ -58,6 +58,8 @@ do
         c) CLIENT_FILE=${OPTARG};;
         # Indicates whether slurm should be used when it is installed.
         s) SLURM=${OPTARG};;
+        # Configuration file path.
+        f) FILE=${OPTARG};;
     esac
 done
 
@@ -70,6 +72,7 @@ echo "DATA_SERVER_FILE=$DATA_SERVER_FILE"
 echo "META_SERVER_FILE=$META_SERVER_FILE"
 echo "SLURM=$SLURM"
 echo "STATUS=$STATUS"
+echo "FILE=$FILE"
 
 ## Checks if the user wants to stop the services.
 if [[ $STATUS = "stop" ]];
@@ -87,15 +90,26 @@ then
     exit 0
 fi
 
-## To know if a configuration file exists in the default paths.
-for FILE in {"/etc/hercules.conf","../conf/hercules.conf","./hercules.conf"}
-do
-    echo $FILE
-    if [ -f "$FILE" ]; then
-        echo "Reading configuration file at $FILE"
-        break
-    fi
-done
+## If a configuration file was not provided.
+if [ -z "$FILE" ]; then
+    ## To know if a configuration file exists in the default paths.
+    for FILE in {"/etc/hercules.conf","../conf/hercules.conf","./hercules.conf"}
+    do
+        echo $FILE
+        if [ -f "$FILE" ]; then
+            break
+        fi
+    done
+fi
+
+if [ -z "$FILE" ]; then
+    echo "No configuration file was provided. Please create one and specify it using -f <file_path>"
+    exit 1
+fi
+
+echo "Reading configuration file at $FILE"
+export HERCULES_CONF="$FILE"
+
 
 ## If slurm is used, then we create a hostfile containing the allocated nodes.
 if [[ $SLURM -eq 1 ]]; then
@@ -229,6 +243,7 @@ echo "[-] Data servers started in $runtime seconds, start=$start, end=$end"
 
 tail -n +$((NUM_METADATA+NUM_DATA+1)) hostfile | head -n $NUM_NODES_FOR_CLIENTS > $H_MPI_HOSTFILE_NAME
 export H_NCPN=$NUM_CLIENTS_PER_NODE
+export H_NNFC=$NUM_NODES_FOR_CLIENTS
 
 ## Search for the mpi distribution installed.
 for MPI_DS in {"openmpi","mpich","impi"}
@@ -245,11 +260,14 @@ case $MPI_DS in
         echo "[+] Option openmpi"
         export H_MPI_ENV_DEF="-x"
         export H_MPI_HOSTFILE_DEF="-hostfile"
+        export H_MPI_PPN="--npernode"
+        #export H_MPI_NNFC=""
     ;;
     "mpich" | "impi")
         echo "[+] Option mpich | impi"
         export H_MPI_ENV_DEF="-env"
         export H_MPI_HOSTFILE_DEF="-f"
+        export H_MPI_PPN="-ppn"
     ;;
     *)
         # Check!

conf/hercules-template.conf

+# Used URI for internal items definition
+URI = imss://
+
+# Block size (in KB)
+BLOCK_SIZE = 512
+
+# Used mount point in the client side
+MOUNT_POINT = /mnt/imss
+
+# Port listening in the metadata node service
+METADATA_PORT = 75000
+
+# Port listening in the data node service
+DATA_PORT =     85000
+
+# Total number of data nodes
+NUM_DATA_SERVERS = 16
+
+# Total number of metadadata nodes
+NUM_META_SERVERS = 1
+
+# Total number of client nodes
+NUM_NODES_FOR_CLIENTS = 16
+
+# Total number of clients per node
+NUM_CLIENTS_PER_NODE = 1
+
+# 1: enables malleability functions
+MALLEABILITY = 0
+UPPER_BOUND_MALLEABILITY = 0
+LOWER_BOUND_MALLEABILITY = 0
+
+# File containing a list of nodes serving as data nodes
+DATA_HOSTFILE = data_hostfile 
+
+# File path of the persistence metadata
+### Tucan
+## METADA_PERSISTENCE_FILE = /home/gsanchez/IMSS/imss/bash/metadata
+### MN4
+## METADA_PERSISTENCE_FILE = /home/uc3m15/uc3m15006/Hercules/imss/bash/metadata
+### Unito
+METADA_PERSISTENCE_FILE = /beegfs/home/javier.garciablas/imss/bash/metadata
+
+# Number of threads attending data requests
+THREAD_POOL = 1
+
+# Maximum size used by the data nodes
+STORAGE_SIZE = 0 # No limit
+
+# Main metadata node hostname
+METADATA_HOST = 
+
+# File containing a list of nodes serving as metadata nodes
+METADATA_HOSTFILE = meta_hostfile
+
+# Debug mode (none or all)
+DEBUG_LEVEL = none

src/workers.c

@@ -1491,7 +1491,7 @@ void *dispatcher(void *th_argv)
 				char mode[MODE_SIZE];
 
 				slog_debug("[DISPATCHER] Waiting for connection requests.");
-				fprintf(stderr, "[DISPATCHER] Waiting for connection requests.\n");
+				// fprintf(stderr, "[DISPATCHER] Waiting for connection requests.\n");
 				sockfd = accept(listenfd, NULL, NULL);
 				ret = recv(sockfd, req, REQUEST_SIZE, MSG_WAITALL);
 

tools/imss_posix.c

@@ -328,43 +328,41 @@ void getConfiguration()
 	/***************************************************************/
 	/******************* PARSE FILE ARGUMENTS **********************/
 	/***************************************************************/
+	int ret = 0;
 
-	char conf_path[PATH_MAX];
-	char abs_exe_path[PATH_MAX];
-	// char abs_exe_path2[PATH_MAX];
-	char *p;
+	char *conf_path;
+	char abs_exe_path[1024];
 	char *aux;
 
-	// readlink("/proc/self/exe", abs_exe_path, PATH_MAX);
-
-	getcwd(abs_exe_path, sizeof(abs_exe_path));
-	// strcpy(abs_exe_path2, abs_exe_path);
-
-	// strcat(abs_exe_path, "/../conf/hercules.conf");
-	// strcat(abs_exe_path2, "./hercules.conf");
-
 	cfg = cfg_init();
-
-	if (getenv("IMSS_CONF") != NULL)
+	conf_path = getenv("IMSS_CONF");
+	if (conf_path != NULL)
 	{
-		cfg_load(cfg, getenv("IMSS_CONF"));
+		ret = cfg_load(cfg, conf_path);
+	}
+	else
+	{
+		ret = 1;
 	}
 
-	// fprintf(stderr, "[Client] Trying to load /etc/hercules.conf\n");
-	strcpy(conf_path, "/etc/hercules.conf");
-	if (cfg_load(cfg, conf_path) > 0)
+	if (ret)
 	{
-		// strcpy(abs_exe_path, "/../conf/hercules.conf");
-		sprintf(conf_path, "%s/%s", abs_exe_path, "../conf/hercules.conf");
-		// fprintf(stderr, "[Client] Trying to load %s\n", abs_exe_path);
+		conf_path = (char *)malloc(sizeof(char) * PATH_MAX);
+		strcpy(conf_path, "/etc/hercules.conf");
 		if (cfg_load(cfg, conf_path) > 0)
 		{
-			// strcpy(abs_exe_path, "./hercules.conf");
-			sprintf(conf_path, "%s", "./hercules.conf");
-			// fprintf(stderr, "[Client] Trying to load %s\n", abs_exe_path2);
+			if(getcwd(abs_exe_path, sizeof(abs_exe_path)) != NULL ) {
+				sprintf(conf_path, "%s/%s", abs_exe_path, "../conf/hercules.conf");
+			} else {
+				sprintf(conf_path, "%s", "./hercules.conf");
+			}
 			if (cfg_load(cfg, conf_path) > 0)
 			{
-				cfg_load(cfg, "hercules.conf");
+				sprintf(conf_path, "%s", "./hercules.conf");
+				if (cfg_load(cfg, conf_path) > 0)
+				{
+					cfg_load(cfg, "hercules.conf");
+				}
 			}
 		}
 	}