summaryrefslogtreecommitdiff
path: root/code/trainer
diff options
context:
space:
mode:
Diffstat (limited to 'code/trainer')
-rw-r--r--code/trainer/Makefile29
-rw-r--r--code/trainer/check_stdp_freq-dep.cpp160
-rw-r--r--code/trainer/check_stdp_freq-dep.h46
-rw-r--r--code/trainer/mem1.cpp412
-rw-r--r--code/trainer/mem1.h72
-rw-r--r--code/trainer/reinforce_synapse.cpp302
-rw-r--r--code/trainer/reinforce_synapse.h55
-rw-r--r--code/trainer/test.cpp13
8 files changed, 1089 insertions, 0 deletions
diff --git a/code/trainer/Makefile b/code/trainer/Makefile
new file mode 100644
index 0000000..1fc48d1
--- /dev/null
+++ b/code/trainer/Makefile
@@ -0,0 +1,29 @@
+# HINT: the paradigm is not to create object files to allow all otpimizations
+# take place even though the code is scattered across many files
+
+CC=g++
+CCFLAGS=-O3 -ggdb
+LDFLAGS=-lpthread -lm
+INCLUDE=-I/home/huwald/src/boost -I../core
+
+BASE_SRC_FILES=../core/model_switch.h ../core/fileutils.h ../core/fileutils.cpp
+
+.PHONY: all clean wordcount
+
+all: reinforce_synapse check_stdp_freq-dep
+
+clean:
+ rm -f *~ trainer massif.*.* reinforce_synapse-* check_stdp_freq-dep-*
+
+.PHONY: reinforce_synapse
+reinforce_synapse: reinforce_synapse-dalif
+reinforce_synapse-%: reinforce_synapse.cpp reinforce_synapse.h ../core/models/%.h $(BASE_SRC_FILES)
+ $(CC) -o $@ $(CCFLAGS) reinforce_synapse.cpp $(INCLUDE) $(LDFLAGS) -DMODEL_`echo $* | tr '[:lower:]' '[:upper:]'`
+
+.PHONY: check_stdp_freq-dep
+check_stdp_freq-dep: check_stdp_freq-dep-dalif
+check_stdp_freq-dep-%: check_stdp_freq-dep.cpp check_stdp_freq-dep.h ../core/models/%.h $(BASE_SRC_FILES)
+ $(CC) -o $@ $(CCFLAGS) check_stdp_freq-dep.cpp $(INCLUDE) $(LDFLAGS) -DMODEL_`echo $* | tr '[:lower:]' '[:upper:]'`
+
+wordcount:
+ wc *h *cpp Makefile
diff --git a/code/trainer/check_stdp_freq-dep.cpp b/code/trainer/check_stdp_freq-dep.cpp
new file mode 100644
index 0000000..f606d81
--- /dev/null
+++ b/code/trainer/check_stdp_freq-dep.cpp
@@ -0,0 +1,160 @@
+#include <stdlib.h>
+#include "fileutils.h"
+#include "math.h"
+#include "unistd.h"
+
+#include "check_stdp_freq-dep.h"
+#include "fileutils.cpp"
+#include "model_switch.h"
+
+using namespace std;
+
+int main(int argc, char **argv) {
+ // check cmd line sanity
+ if (argc != 5) {
+ fprintf(stderr, "Wrong argument count\n\n"
+ "Call format:\n"
+ "%s\n\t"
+ "performance out\n\t"
+ "trace cmd out\n\t"
+ "global out\n\t"
+ "spike out\n\t"
+ "\n"
+ "Special names allowed:\n\t- (standart input)\n\t0 (/dev/null)\n", argv[0]);
+ return -1;
+ }
+
+ Trainer *t = new Trainer(argc, argv);
+ t->run();
+
+ pthread_join(t->thread_write, NULL);
+}
+
+Trainer::Trainer(int argc, char** argv) {
+ // init vars
+ currentEpoch = 0;
+ epochDuration = 10.0; // [s]
+ neurons = 1000; // number of neurons to send noise to
+ voltage = 0.1; // [V]
+ md = 1.2; // 10.0;
+ mss = 1.1; //1.2;
+ fs = 100; // number of frequencies to try
+ frd = 1.0; // relative difference between to frequencies (f_i+1 = frd * f_i)
+ fad = 0.5; // absolute difference between to frequencies (f_i+1 = fad + f_i)
+
+ // open all file descriptors in an order complementary to the simulators one
+ // to avoid deadlocks
+ fd_spike_out = fd_magic(argv[4], true);
+ fd_global_out = fd_magic(argv[3], true);
+ fd_performance_out = fd_magic(argv[1], true);
+ fd_trace_out = fd_magic(argv[2], true);
+
+ // create read and write threads
+ pthread_create(&thread_write, NULL, (void* (*)(void*)) &write_spikes, this);
+}
+
+void Trainer::run() {
+ char *str_trace = "%f; synapse\n";
+
+ // init global sim variables
+ MS_Global msg;
+ msg_init(msg);
+ msg.dopamin_level = 0.0;
+
+ double ta = 0.009821, //0.0088541,
+ la = 0.140249; // 0.126445;
+
+ /*
+ // loop over both vars to examine
+ for (msg.stdp_tau_plus = msg.stdp_tau_minus / md;
+ msg.stdp_tau_plus <= msg.stdp_tau_minus * md;
+ msg.stdp_tau_plus *= mss) {
+ for (msg.stdp_lambda_plus = msg.stdp_lambda_minus / md;
+ msg.stdp_lambda_plus <= msg.stdp_lambda_minus * md;
+ msg.stdp_lambda_plus *= mss) {*/
+ // loop over both vars to examine
+ for (msg.stdp_tau_plus = ta / md;
+ msg.stdp_tau_plus <= ta * md;
+ msg.stdp_tau_plus *= mss) {
+ for (msg.stdp_lambda_plus = la / md;
+ msg.stdp_lambda_plus <= la * md;
+ msg.stdp_lambda_plus *= mss) {
+
+ // print the parameters to the performance output
+ msg_print(msg, fd_performance_out);
+ fprintf(fd_performance_out, "\n");
+
+ // print the global params
+ fprintf(fd_global_out, "%f, ", currentEpoch * epochDuration);
+ msg_print(msg, fd_global_out);
+ fprintf(fd_global_out, "\n");
+
+ // let the simulation proceed
+ fprintf(fd_trace_out, str_trace, epochDuration);
+ currentEpoch++;
+
+ // repeat this 2*n-1 times (n=number of different frequency trials)
+ for (int i=0; i < 2*fs-1; i++) {
+ fprintf(fd_trace_out, "\n");
+ currentEpoch++;
+ }
+ }
+ }
+
+ fclose(fd_trace_out);
+ fclose(fd_global_out);
+}
+
+// ---- send indepenent poisson noise w/ increasing fequency----
+void *write_spikes(Trainer *t) {
+ // calculate how often we have to try all frequencies (=outer loop)
+ // WARN: ignore minor numerical instabilities
+ int max = (int) floor(2.0 * log(t->md) / log(t->mss) ) + 1;
+ max *= max; // there are two nested loops of the same size
+
+ double time = 0.0; // global time (that one send to the simulator)
+
+ // for each paramter config (set in the main routine)
+ for (int i=0; i<max; i++) {
+
+ double freq = 1.0;
+
+ // examine a set of frequencies
+ for (int j=0; j < t->fs; j++) {
+ // send out the spikes
+ double localtime = 0.0;
+ double nextRefSpike = 0.0;
+ double refFreq = 10.0; // [Hz]
+ int dst = -1;
+ while (localtime < t->epochDuration) {
+ // starting with the second call ...
+ if (dst != -1) {
+ // check if we have to send a spike to the ref neuron
+ if (localtime > nextRefSpike) {
+ fprintf(t->fd_spike_out, "%f, %d, %f\n", time + nextRefSpike, 0, t->voltage);
+ nextRefSpike += 1.0 / refFreq;
+ }
+
+ // send spike to the simulator
+ fprintf(t->fd_spike_out, "%f, %d, %f\n", time + localtime, dst, t->voltage);
+ }else{
+ }
+
+ localtime -= log(1.0 - drand48()) / (freq * t->neurons); // possion distributed spike timing
+ dst = 1 + rand() % (t->neurons - 1); // random neuron (except reference neuron 0)
+ }
+
+ // increase time (twice because of the silence period after each noise period)
+ time = (i * t->fs + j) * 2.0 * t->epochDuration;
+
+ // increase frequency
+ freq *= t->frd;
+ freq += t->fad;
+ }
+ }
+
+ // close fd because fscanf sucks
+ fclose(t->fd_spike_out);
+
+ return NULL;
+}
diff --git a/code/trainer/check_stdp_freq-dep.h b/code/trainer/check_stdp_freq-dep.h
new file mode 100644
index 0000000..cf429b6
--- /dev/null
+++ b/code/trainer/check_stdp_freq-dep.h
@@ -0,0 +1,46 @@
+#ifndef TRAINER_H
+#define TRAINER_H
+
+#include <stdio.h>
+#include <pthread.h>
+#include <map>
+#include <queue>
+#include "boost/tuple/tuple.hpp"
+
+
+using namespace std;
+
+class Trainer {
+ public:
+ FILE *fd_spike_out,
+ *fd_global_out,
+ *fd_trace_out,
+ *fd_performance_out;
+
+ // init stuff
+ Trainer(int argc, char** argv);
+
+ // main routine
+ void run();
+
+ // state vars
+ long currentEpoch;
+
+ // thread related
+ pthread_t thread_write;
+
+ // configuration
+ double md; // multiplicative difference (>1)
+ double mss; // multiplicative step size (>1)
+ double fs, frd, fad; // number of frequency steps and relative step size
+ double epochDuration;
+ double voltage; // per outgoing (random) spike
+ long neurons;
+};
+
+// seperate thread to read all spikes neccessary because reading and
+// writing to these descriptors could block and thus cause a deadlock
+void *read_spikes(Trainer *t);
+void *write_spikes(Trainer *t);
+
+#endif // TRAINER_H
diff --git a/code/trainer/mem1.cpp b/code/trainer/mem1.cpp
new file mode 100644
index 0000000..3b522b4
--- /dev/null
+++ b/code/trainer/mem1.cpp
@@ -0,0 +1,412 @@
+#include <stdlib.h>
+#include "fileutils.h"
+#include "math.h"
+
+#include "reinforce_synapse.h"
+#include "fileutils.cpp"
+#include "model_switch.h"
+
+using namespace std;
+
+int main(int argc, char **argv) {
+ // check cmd line sanity
+ if (argc != 7) {
+ fprintf(stderr, "Wrong argument count\n\n"
+ "Call format:\n"
+ "%s\n\t"
+ "performance out\n\t"
+ "trace cmd out\n\t"
+ "global out\n\t"
+ "global in\n\t"
+ "spike out\n\t"
+ "spike in\n\t"
+ "\n"
+ "Special names allowed:\n\t- (standart input)\n\t0 (/dev/null)\n", argv[0]);
+ return -1;
+ }
+
+ Trainer *t = new Trainer(argc, argv);
+ t->run();
+}
+
+//===== Initialisation =====================================
+
+Trainer::Trainer(int argc, char** argv) {
+ initConfig();
+ initState();
+ initGroups(); // determine input and output neurons
+
+ initFiles();
+ initThreads();
+}
+
+void Trainer::initConfig() {
+ neurons = 1000;
+ neuronsPerSymbol = 200;
+ noiseFreq = 10.0; // [Hz]
+ noiseVoltage = 0.03; // [V]
+ reward = 0.1;
+
+ epochDuration = 1.0; // [s]
+ numTrials = 1000;
+ numSymbols = 2;
+ //readoutDelay = 1;
+ refractoryPeriods = 3;
+}
+
+void Trainer::initState() {
+ dopamin_level = 0.0;
+ currentTrial = 0;
+ state = 0;
+
+ msg_init(msg);
+ msg.dopamin_level = dopamin_level;
+
+ groupFreq.resize(numSymbols);
+ for (int i=0; i<numSymbols; i++) {
+ groupFreq[i] = 0;
+ }
+}
+
+void Trainer::initGroups() {
+ ioNeurons[i] = new set<int>();
+ for (int j=0; j<neuronsPerSymbol;) {
+ int n = rand() % numNeurons;
+ if (!isNeurons[i]->count(n)) {
+ ioNeurons[i]->insert(n);
+ j++;
+ }
+ }
+}
+
+void Trainer::initFiles() {
+ // open all file descriptors in an order complementary to the simulators one
+ // to avoid deadlocks
+ fd_spike_in = fd_magic(argv[6], false);
+ fd_global_in = fd_magic(argv[4], false);
+ fd_spike_out = fd_magic(argv[5], true);
+ fd_global_out = fd_magic(argv[3], true);
+ fd_performance_out = fd_magic(argv[1], true);
+ fd_trace_out = fd_magic(argv[2], true);
+}
+
+void Trainer::initThreads() {
+ // init locks
+ pthread_mutex_init(&incomingSpikeLock, NULL);
+ pthread_mutex_init(&writerLock, NULL);
+
+ // create read and write threads
+ pthread_create(&thread_read, NULL, (void* (*)(void*)) &read_spikes, this);
+ pthread_create(&thread_write, NULL, (void* (*)(void*)) &write_spikes, this);
+}
+
+//===== Core trainer ====================================
+
+void Trainer::pushGlobal(double time) {
+ fprintf(fd_global_out, "%f, ", time);
+ msg_print(msg, fd_global_out);
+ fprintf(fd_global_out, "\n");
+ fflush(fd_global_out);
+}
+
+// hint time is delta time!
+void Trainer::pushTrace(double time) {
+ const char *str_trace = "%f; spikes (0; 1); global; neuron (0; 1); synapse (0; 1)\n";
+ fprintf(fd_trace_out, str_trace, time);
+ fflush(fd_trace_out);
+}
+
+bool Trainer::readGlobal() {
+ double _foo_dbl;
+ char str_raw[128],
+ str_msg[128];
+ str_raw[0] = 0;
+
+ // read a single line
+ if (fgets((char*) str_raw, 128, fd_global_in) == NULL) {
+ fprintf(stderr, "ERROR: global status file descriptor from simulator closed unexpectedly\n");
+ return false;
+ }
+
+ // parse it
+ if ((sscanf((char*) str_raw, "%lf, %[^\n]\n", &_foo_dbl, (char*) str_msg) != 2)
+ || (!msg_parse(msg, (char*) str_msg))) {
+ fprintf(stderr, "ERROR: reading global status from simulator failed\n\t\"%s\"\n", (char*) str_raw);
+ return false;
+ }
+
+ return true;
+}
+
+void binIncomingSpikes() {
+ // reset bins
+ for (int i=0; i<groupFreq.size(); i++)
+ groupFreq[i] = 0;
+
+ // lock spike queue
+ pthread_yield(); // give the spike reading thread chance to finish ... this is not more than ugly semifix wrong par!
+ pthread_mutex_lock(&incomingSpikeLock);
+
+ // read all spikes in the correct time window
+ while ((!incomingSpikes.empty()) && (incomingSpikes.front().get<0>() <= currentEpoch * epochDuration)) {
+ // drop event out of queue
+ SpikeEvent se = incomingSpikes.front();
+ double time = se.get<0>();
+ int neuron = se.get<1>();
+ incomingSpikes.pop();
+
+ // check if it belongs to the previous bin (and ignore it if this is the case)
+ if (time < (currentEpoch - 1) * epochDuration) {
+ fprintf(stderr, "WARN: spike reading thread to slow; unprocessed spike of the past discovered\n%f\t%f\t%d\t%f\n",
+ time, (double) (currentEpoch - 1) * epochDuration, currentEpoch, epochDuration);
+ continue;
+ }
+
+ // check membership in each group and increase group frequency
+ for (int i=0; i < ioNeurons.size(); i++)
+ if (ioNeuros[i]->count(neuron))
+ groupFreq[i]++;
+ }
+
+ pthread_mutex_unlock(&incomingSpikeLock);
+}
+
+void Trainer::addBaselineSpikes() {
+}
+
+void Trainer::addSymbolSpikes() {
+
+}
+
+
+double Trainer::calcSignalStrength() {
+ if (symbolHist.empty()) {
+ fprintf(stderr, "Writer thread is too slow; missed the current symbol\n");
+ exit(-1);
+ }
+
+ int fs, fn = 0; // freq signal, freq noise
+ for (int i=0; i<numSymbols; i++) {
+ if (i == symbolHist.front()) {
+ fs = groupFreq[i];
+ }else{
+ fm = fmax(fm, groupFreq[i]);
+ }
+ }
+
+ if (fn == 0) {
+ return fs * INFINITY;
+ }else{
+ return ((double) fs) / fn;
+ }
+}
+
+void Trainer::run() {
+ // rough description of this function
+ // . start an epoch
+ // . wait for it's end
+ // . process incomig spikes (binning)
+ // . select if a reward takes place
+ // . print reward value
+ // . send out the reward signal
+
+ // send out the full trace command once (later it will be repeated by sending newline)
+ pushTrace(epochDuration);
+
+ // send the first two global states (at t=0 and t=1.5 [bintime] to allow the simulation to
+ // be initialized (before the causality of the loop below is met)
+ pushGlobal(0.0);
+ msg_process(msg, 1.5 * epochDuration);
+ dopamin_level = msg.dopamin_level;
+ pushGlobal(1.5 * epochDuration);
+
+ // loop until the experiment is done
+ for (; currentEpoch * epochDuration < entireDuration; currentEpoch++) {
+
+ // send a new trace command (do it as early as possible although it is
+ // only executed after the new global is send out at the bottom of this loop)
+ if ((currentEpoch + 2) * epochDuration < entireDuration) {
+ // repeat the previous trace command
+ fprintf(fd_trace_out, "\n");
+ fflush(fd_trace_out);
+ }else{
+ pushTrace(entireDuration - (currentEpoch + 1) * epochDuration);
+ }
+
+ // send new spikes
+ pthread_mutex_lock(&outgoingSpikeLock);
+ addBaselineSpikes();
+ if (state == 0) addSymbolSpikes();
+ pthread_cond_signal(&outgoingSpikeCond);
+ pthread_mutex_unlock(&outgoingSpikeLock);
+
+ // wait for the end of the epoch (by reading the global state resulting from it)
+ if (!readGlobal())
+ break;
+
+ // process incomig spikes (binning) of the previous epoch
+ if (currentEpoch > 0)
+ binIncomingSpikes();
+
+ // proceed the global state to keep it in sync with the simulator's global state
+ // the local dopamin level is kept seperately and aged only one epochDuration to
+ // avoid oscillation effects in dopamin level
+ msg_process(msg, 1.5 * epochDuration);
+ dopamin_level *= exp( - epochDuration / msg.dopamin_tau );
+
+ // do various actions depeding on state (thus lock mutex of the writer thread)
+
+
+ switch (state) {
+ case 0: // a signal is sent
+ state++;
+
+ case 1: // we are waiting for the signal to be reproduce
+ // get fraction of the current symbol's freq compared to the strongest wrong symbol
+ double ss = calcSignalStrength();
+
+ // check if the reward condition is met
+ if (ss > 1) {
+ dopmain_level += reward;
+ }else{
+ state++; // lost signal -> next state (and finally a new trial)
+ currentSymbol = rand() % numSymbols; // determine new symbol to display
+ }
+
+ break;
+
+ default: // the signal has been lost (in the last round); refractory time
+ ++state %= refractoryPeriods;
+ }
+
+ /*if ((currentEpoch > 1) && ((*neuronFreq[0])[0] > 0) && ((*neuronFreq[1])[1] > 0)) {
+ dopamin_level += da_single_reward;
+ fprintf(fd_performance_out, "+");
+ }else{
+ fprintf(fd_performance_out, "-");
+ }*/
+
+ // performance and "debug" output
+ if (currentEpoch > 1) {
+ //fprintf(fd_performance_out, "\n");
+ fprintf(fd_performance_out, "\t%f\t%d\t%d\n", dopamin_level, (*neuronFreq[0])[0], (*neuronFreq[1])[1]);
+ }else{
+ // fake output as acutal data is not available, yet
+ fprintf(fd_performance_out, "\t%f\t%d\t%d\n", dopamin_level, (int) 0, (int) 0);
+ }
+
+ // set the new DA level
+ msg.dopamin_level = dopamin_level;
+
+ // print new global state
+ // (do this even if there has been no evaluation of the performance yet,
+ // because it is neccessary for the simulator to proceed)
+ pushGlobal(((double) currentEpoch + 2.5) * epochDuration);
+ }
+
+ fclose(fd_trace_out);
+
+ // terminate child threads
+ pthread_cancel(thread_read);
+ pthread_cancel(thread_write);
+}
+
+void *read_spikes(Trainer *t) {
+ double lastSpike = -INFINITY; // used to check if the spikes are coming in order
+
+ // read spikes until eternity
+ while (!feof(t->fd_spike_in)) {
+ // read one line from stdin (blocking)
+ char buf[128];
+ if (fgets((char*) buf, 128, t->fd_spike_in) == NULL) continue; // this should stop the loop because of EOF
+
+ // parse the input
+ double time, current;
+ int neuron;
+ switch (sscanf((char*) buf, "%lf, %d, %lf\n", &time, &neuron, &current)) {
+ case 3:
+ // format is ok, continue
+ break;
+ default:
+ // format is wrong, stop
+ fprintf(stderr, "ERROR: malformatted incoming spike:\n\t%s\n", &buf);
+ return NULL;
+ }
+
+ if (lastSpike > time) {
+ fprintf(stderr, "WARN: out of order spike detected (coming from simulator)\n\t%f\t%d\n", time, neuron);
+ continue;
+ }
+
+ lastSpike = time;
+
+ // add the spike to the queue of spikes
+ pthread_mutex_lock(&(t->incomingSpikeLock));
+ t->incomingSpikes.push(boost::make_tuple(time, neuron, current));
+ pthread_mutex_unlock(&(t->incomingSpikeLock));
+ }
+
+ // we shouldn't reach this point in a non-error case
+ fprintf(stderr, "ERROR: EOF in incoming spike stream\n");
+ // TODO: kill entire programm
+ return NULL;
+}
+
+void *write_spikes(Trainer *t) {
+ // at the moment: generate noise until the file descriptor blocks
+ double time = 0.0;
+
+ // PAR HINT:
+ // loop until exactly one spike after the entire duration is send out
+ // this will block on full buffer on the file descriptor and thus keep
+ // the thread busy early enough
+
+
+ /* // ---- send 100% dependent spike train ---
+ time = 0.005;
+ while (time <= t->entireDuration) {
+ fprintf(t->fd_spike_out, "%f, %d, %f\n", time, 0, 1.0);
+ time += 0.012;
+ fprintf(t->fd_spike_out, "%f, %d, %f\n", time, 1, 1.0);
+ time += 1.0;
+ }*/
+
+
+ /* // ---- send indepenent poisson noise ----
+ while (time <= t->entireDuration) {
+ // calc timing, intensity and destination of the spike
+ // HINT:
+ // * log(...) is negative
+ // * drand48() returns something in [0,1), to avoid log(0) we transform it to (0,1]
+ time -= log(1.0 - drand48()) / (t->freq * t->neurons);
+ int dst = rand() % t->neurons;
+ double current = t->voltage;
+
+ // send it to the simulator
+ fprintf(t->fd_spike_out, "%f, %d, %f\n", time, dst, current);
+ }*/
+
+ // ---- send indepenent poisson noise w7 increasing fequency----
+ double blafoo = 0;
+ t->freq = 1.0;
+ while (time <= t->entireDuration) {
+ if (time - blafoo > 100.0) {
+ blafoo += 200.0;
+ t->freq += 1.0;
+ time += 100.0; // time jump to let ET recover to zero
+ }
+ // calc timing, intensity and destination of the spike
+ // HINT:
+ // * log(...) is negative
+ // * drand48() returns something in [0,1), to avoid log(0) we transform it to (0,1]
+ time -= log(1.0 - drand48()) / (t->freq * t->neurons);
+ int dst = rand() % t->neurons;
+ double current = t->voltage;
+
+ // send it to the simulator
+ fprintf(t->fd_spike_out, "%f, %d, %f\n", time, dst, current);
+ }
+
+ // close fd because fscanf sucks
+ fclose(t->fd_spike_out);
+}
diff --git a/code/trainer/mem1.h b/code/trainer/mem1.h
new file mode 100644
index 0000000..31fc2b0
--- /dev/null
+++ b/code/trainer/mem1.h
@@ -0,0 +1,72 @@
+#ifndef TRAINER_H
+#define TRAINER_H
+
+#include <stdio.h>
+#include <pthread.h>
+#include <map>
+#include <queue>
+#include "boost/tuple/tuple.hpp"
+
+
+using namespace std;
+
+class Trainer {
+ public:
+ FILE *fd_spike_in,
+ *fd_spike_out,
+ *fd_global_out,
+ *fd_global_in,
+ *fd_trace_out,
+ *fd_performance_out;
+
+ // init stuff
+ Trainer(int argc, char** argv);
+ void initConfig();
+ void initState();
+ void initGroups();
+ void initFiles();
+ void initThreads();
+
+ // main routine
+ void run();
+
+ // state vars
+ long currentTrial;
+ vector<int> groupFreq; // spikes fired during epoch for each symbol (=group of neurons)
+ vector< set<int>* > ioNeurons; // a set of neurons for each symbol to which the symbol is written and from wich it is read later
+ //queue< int > symbolHist; // stores the symbols displayed; written by thread_write, read by main thread
+ int currentSymbol;
+ double dopamin_level;
+ int state;
+ MS_Global msg;
+
+ // synchronisation vars
+ typedef boost::tuple<double, int, double> SpikeEvent; // <what time, wich neuron, current>
+ queue<SpikeEvent> incomingSpikes;
+ // TODO: outgoingSpikes;
+ pthread_mutex_t incomingSpikeLock, writerLock;
+ // TODO: , outgoingSpikeLock; (including a condition for the writer to wakeup upon if a previously empyt queue has been filled)
+ pthread_t thread_read, thread_write;
+
+ // configuration
+ // network
+ long neurons; // total number of neurons
+ long neuronsPerSymbol;
+ double noiseFreq; // of poisson noise (per neuron)
+ double noiseVoltage; // per noise spike
+ double reward; // per succesful trial
+
+ // learning task
+ double epochDuration; // a trial consists of several epochs
+ long numTrials;
+ int numSymbols; // number of different things to remember
+ //int readoutDelay; // number of epochs between symbol-write and symbol-read-epoch
+ int refractoryPeriods; // how many epochs to wait after a trial finished until we start with a new trial
+};
+
+// seperate thread to read all spikes neccessary because reading and
+// writing to these descriptors could block and thus cause a deadlock
+void *read_spikes(Trainer *t);
+void *write_spikes(Trainer *t);
+
+#endif // TRAINER_H
diff --git a/code/trainer/reinforce_synapse.cpp b/code/trainer/reinforce_synapse.cpp
new file mode 100644
index 0000000..bf6fc7f
--- /dev/null
+++ b/code/trainer/reinforce_synapse.cpp
@@ -0,0 +1,302 @@
+#include <stdlib.h>
+#include "fileutils.h"
+#include "math.h"
+
+#include "reinforce_synapse.h"
+#include "fileutils.cpp"
+#include "model_switch.h"
+
+using namespace std;
+
+int main(int argc, char **argv) {
+ // check cmd line sanity
+ if (argc != 7) {
+ fprintf(stderr, "Wrong argument count\n\n"
+ "Call format:\n"
+ "%s\n\t"
+ "performance out\n\t"
+ "trace cmd out\n\t"
+ "global out\n\t"
+ "global in\n\t"
+ "spike out\n\t"
+ "spike in\n\t"
+ "\n"
+ "Special names allowed:\n\t- (standart input)\n\t0 (/dev/null)\n", argv[0]);
+ return -1;
+ }
+
+ Trainer *t = new Trainer(argc, argv);
+ t->run();
+ // TODO: finalize
+}
+
+Trainer::Trainer(int argc, char** argv) {
+ // init vars
+ currentEpoch = 0;
+ dopamin_level = 0.0;
+
+ epochDuration = 0.01; // [s]
+ //epochDuration = 1.0; // [s]
+ entireDuration = 20000.0; // [s]
+ neurons = 2; // number of neurons to send noise to
+ freq = 1.0; // [Hz] per Neuron
+ voltage = 0.1; // [V]
+ da_single_reward = 0.01;
+
+ neuronFreq[0] = (map<int, int>*) NULL;
+ neuronFreq[1] = (map<int, int>*) NULL;
+
+ // open all file descriptors in an order complementary to the simulators one
+ // to avoid deadlocks
+ fd_spike_in = fd_magic(argv[6], false);
+ fd_global_in = fd_magic(argv[4], false);
+ fd_spike_out = fd_magic(argv[5], true);
+ fd_global_out = fd_magic(argv[3], true);
+ fd_performance_out = fd_magic(argv[1], true);
+ fd_trace_out = fd_magic(argv[2], true);
+
+ // init locks
+ pthread_mutex_init(&incomingSpikeLock, NULL);
+
+ // create read and write threads
+ pthread_create(&thread_read, NULL, (void* (*)(void*)) &read_spikes, this);
+ pthread_create(&thread_write, NULL, (void* (*)(void*)) &write_spikes, this);
+}
+
+void Trainer::run() {
+ // start an epoch
+ // wait for it's end
+ // process incomig spikes (binning)
+ // select if a reward takes place
+ // print reward value (TODO: into a seperate, externally given file descriptor)
+ // send out the reward signal
+
+ char *str_trace = "%f; spikes (0; 1); global; neuron (0; 1); synapse (0; 1)\n";
+
+ // send out the full trace commande once (later it will be repeated by sending newline)
+ fprintf(fd_trace_out, str_trace, epochDuration);
+ fflush(fd_trace_out);
+
+ // send the first two global states (at t=0 and t=1.5 [bintime] to allow the simulation to
+ // be initialized (before the causality of the loop below is met)
+ MS_Global msg;
+ msg_init(msg);
+ msg.dopamin_level = dopamin_level;
+
+ // set the tau-levels like in Izhi's network
+ msg.stdp_tau_minus = 1.5 * msg.stdp_tau_plus;
+ msg.stdp_lambda_plus = msg.stdp_lambda_minus;
+
+ fprintf(fd_global_out, "0.0, ");
+ msg_print(msg, fd_global_out);
+ fprintf(fd_global_out, "\n");
+
+ msg_process(msg, 1.5 * epochDuration);
+ dopamin_level = msg.dopamin_level;
+
+
+ fprintf(fd_global_out, "%f, ", 1.5 * epochDuration);
+ msg_print(msg, fd_global_out);
+ fprintf(fd_global_out, "\n");
+
+ fflush(fd_global_out);
+
+ // loop until the experiment is done
+ for (; currentEpoch * epochDuration < entireDuration; currentEpoch++) {
+ // send a new trace command (do it as early as possible although it is
+ // only executed after the new global is send out at the bottom of this loop)
+ if ((currentEpoch + 2) * epochDuration < entireDuration) {
+ // repeat the previous trace command
+ fprintf(fd_trace_out, "\n");
+ }else{
+ fprintf(fd_trace_out, str_trace, entireDuration - (currentEpoch + 1) * epochDuration);
+ }
+ fflush(fd_trace_out);
+
+ // wait for the end of the epoch (by reading the global state resulting from it)
+ char str_raw[128], str_msg[128]; str_raw[0] = 0;
+ double _foo_dbl;
+ if (fgets((char*) str_raw, 128, fd_global_in) == NULL) {
+ fprintf(stderr, "ERROR: global status file descriptor from simulator closed unexpectedly\n");
+ break;
+ }
+ if ((sscanf((char*) str_raw, "%lf, %[^\n]\n", &_foo_dbl, (char*) str_msg) != 2)
+ || (!msg_parse(msg, (char*) str_msg))) {
+ fprintf(stderr, "ERROR: reading global status from simulator failed\n\t\"%s\"\n", (char*) str_raw);
+ break;
+ }
+
+ // process incomig spikes (binning) of the previous epoch
+ if (currentEpoch > 0) {
+ // shift the bins
+ if (neuronFreq[0]) {
+ delete neuronFreq[0];
+ neuronFreq[0] = neuronFreq[1];
+ }else{
+ neuronFreq[0] = new map<int, int>();
+ }
+ neuronFreq[1] = new map<int, int>();
+
+ // read all spikes in the correct time window
+ pthread_mutex_lock(&incomingSpikeLock);
+ while ((!incomingSpikes.empty()) && (incomingSpikes.front().get<0>() <= currentEpoch * epochDuration)) {
+ // drop event out of queue
+ SpikeEvent se = incomingSpikes.front();
+ double time = se.get<0>();
+ int neuron = se.get<1>();
+ incomingSpikes.pop();
+
+ // check if it belongs to the previous bin (and ignore it if this is the case)
+ if (time < (currentEpoch - 1) * epochDuration) {
+ fprintf(stderr, "WARN: spike reading thread to slow; unprocessed spike of the past discovered\n%f\t%f\t%d\t%f\n", time, (double) (currentEpoch - 1) * epochDuration, currentEpoch, epochDuration);
+ continue;
+ }
+
+ // increment the frequency counter (relies on int being default constructable to value 0)
+ (*neuronFreq[1])[neuron]++;
+ }
+
+ pthread_mutex_unlock(&incomingSpikeLock);
+ }
+
+ // proceed the global state to keep it in sync with the simulator's global state
+ // the local dopamin level is kept seperately and aged only one epochDuration to
+ // avoid oscillation effects in dopamin level
+ msg_process(msg, 1.5 * epochDuration);
+ dopamin_level *= exp( - epochDuration / msg.dopamin_tau );
+
+ // select if the reward takes place
+ if ((currentEpoch > 1) && ((*neuronFreq[0])[0] > 0) && ((*neuronFreq[1])[1] > 0)) {
+ dopamin_level += da_single_reward;
+ fprintf(fd_performance_out, "+");
+ }else{
+ fprintf(fd_performance_out, "-");
+ }
+
+ if (currentEpoch > 1) {
+ //fprintf(fd_performance_out, "\n");
+ fprintf(fd_performance_out, "\t%f\t%d\t%d\n", dopamin_level, (*neuronFreq[0])[0], (*neuronFreq[1])[1]);
+ }else{
+ // fake output as acutal data i not available, yet
+ fprintf(fd_performance_out, "\t%f\t%d\t%d\n", dopamin_level, (int) 0, (int) 0);
+ }
+
+ // set the new DA level
+ msg.dopamin_level = dopamin_level;
+
+ // print new global state
+ // (do this even if there has been no evaluation of the performance yet,
+ // because it is neccessary for the simulator to proceed)
+
+ fprintf(fd_global_out, "%f, ", ((double) currentEpoch + 2.5) * epochDuration);
+ msg_print(msg, fd_global_out);
+ fprintf(fd_global_out, "\n");
+ fflush(fd_global_out);
+ }
+
+ fclose(fd_trace_out);
+
+ // terminate child threads
+ pthread_cancel(thread_read);
+ pthread_cancel(thread_write);
+}
+
+void *read_spikes(Trainer *t) {
+ double lastSpike = -INFINITY; // used to check if the spikes are coming in order
+
+ // read spikes until eternity
+ while (!feof(t->fd_spike_in)) {
+ // read one line from stdin (blocking)
+ char buf[128];
+ if (fgets((char*) buf, 128, t->fd_spike_in) == NULL) continue; // this should stop the loop because of EOF
+
+ // parse the input
+ double time, current;
+ int neuron;
+ switch (sscanf((char*) buf, "%lf, %d, %lf\n", &time, &neuron, &current)) {
+ case 3:
+ // format is ok, continue
+ break;
+ default:
+ // format is wrong, stop
+ fprintf(stderr, "ERROR: malformatted incoming spike:\n\t%s\n", &buf);
+ return NULL;
+ }
+
+ if (lastSpike > time) {
+ fprintf(stderr, "WARN: out of order spike detected (coming from simulator)\n\t%f\t%d\n", time, neuron);
+ continue;
+ }
+
+ lastSpike = time;
+
+ // add the spike to the queue of spikes
+ pthread_mutex_lock(&(t->incomingSpikeLock));
+ t->incomingSpikes.push(boost::make_tuple(time, neuron, current));
+ pthread_mutex_unlock(&(t->incomingSpikeLock));
+ }
+
+ // we shouldn't reach this point in a non-error case
+ fprintf(stderr, "ERROR: EOF in incoming spike stream\n");
+ // TODO: kill entire programm
+ return NULL;
+}
+
+void *write_spikes(Trainer *t) {
+ // at the moment: generate noise until the file descriptor blocks
+ double time = 0.0;
+
+ // PAR HINT:
+ // loop until exactly one spike after the entire duration is send out
+ // this will block on full buffer on the file descriptor and thus keep
+ // the thread busy early enough
+
+
+ /* // ---- send 100% dependent spike train ---
+ time = 0.005;
+ while (time <= t->entireDuration) {
+ fprintf(t->fd_spike_out, "%f, %d, %f\n", time, 0, 1.0);
+ time += 0.012;
+ fprintf(t->fd_spike_out, "%f, %d, %f\n", time, 1, 1.0);
+ time += 1.0;
+ }*/
+
+
+ // ---- send indepenent poisson noise ----
+ while (time <= t->entireDuration) {
+ // calc timing, intensity and destination of the spike
+ // HINT:
+ // * log(...) is negative
+ // * drand48() returns something in [0,1), to avoid log(0) we transform it to (0,1]
+ time -= log(1.0 - drand48()) / (t->freq * t->neurons);
+ int dst = rand() % t->neurons;
+ double current = t->voltage;
+
+ // send it to the simulator
+ fprintf(t->fd_spike_out, "%f, %d, %f\n", time, dst, current);
+ }
+
+ /*// ---- send indepenent poisson noise w7 increasing fequency----
+ double blafoo = 0;
+ t->freq = 1.0;
+ while (time <= t->entireDuration) {
+ if (time - blafoo > 100.0) {
+ blafoo += 200.0;
+ t->freq += 1.0;
+ time += 100.0; // time jump to let ET recover to zero
+ }
+ // calc timing, intensity and destination of the spike
+ // HINT:
+ // * log(...) is negative
+ // * drand48() returns something in [0,1), to avoid log(0) we transform it to (0,1]
+ time -= log(1.0 - drand48()) / (t->freq * t->neurons);
+ int dst = rand() % t->neurons;
+ double current = t->voltage;
+
+ // send it to the simulator
+ fprintf(t->fd_spike_out, "%f, %d, %f\n", time, dst, current);
+ }*/
+
+ // close fd because fscanf sucks
+ fclose(t->fd_spike_out);
+}
diff --git a/code/trainer/reinforce_synapse.h b/code/trainer/reinforce_synapse.h
new file mode 100644
index 0000000..46b0083
--- /dev/null
+++ b/code/trainer/reinforce_synapse.h
@@ -0,0 +1,55 @@
+#ifndef TRAINER_H
+#define TRAINER_H
+
+#include <stdio.h>
+#include <pthread.h>
+#include <map>
+#include <queue>
+#include "boost/tuple/tuple.hpp"
+
+
+using namespace std;
+
+class Trainer {
+ public:
+ FILE *fd_spike_in,
+ *fd_spike_out,
+ *fd_global_out,
+ *fd_global_in,
+ *fd_trace_out,
+ *fd_performance_out;
+
+ // init stuff
+ Trainer(int argc, char** argv);
+
+ // main routine
+ void run();
+
+ // state vars
+ long currentEpoch;
+ map<int, int> *neuronFreq[2]; // stores if a surveilled neuron fired during the current or last epoch
+ double dopamin_level;
+
+ // synchronisation vars
+ typedef boost::tuple<double, int, double> SpikeEvent; // <what time, wich neuron, current>
+ queue<SpikeEvent> incomingSpikes;
+ // TODO: outgoingSpikes;
+ pthread_mutex_t incomingSpikeLock;
+ // TODO: , outgoingSpikeLock; (including a condition for the writer to wakeup upon if a previously empyt queue has been filled)
+ pthread_t thread_read, thread_write;
+
+ // configuration
+ double epochDuration;
+ double entireDuration;
+ double freq; // of outgoing noise per neuron
+ double voltage; // per outgoing (random) spike
+ long neurons;
+ double da_single_reward;
+};
+
+// seperate thread to read all spikes neccessary because reading and
+// writing to these descriptors could block and thus cause a deadlock
+void *read_spikes(Trainer *t);
+void *write_spikes(Trainer *t);
+
+#endif // TRAINER_H
diff --git a/code/trainer/test.cpp b/code/trainer/test.cpp
new file mode 100644
index 0000000..ceb2484
--- /dev/null
+++ b/code/trainer/test.cpp
@@ -0,0 +1,13 @@
+#include <stdio.h>
+#include <math.h>
+
+int main() {
+ fprintf("aaa\n");
+ while (true);/*
+ double d=0;
+ printf("aa %lf\n", d);
+ d = 0;
+ scanf("%lf\n", &d);
+ printf("%lf\n", d);*/
+ return 0;
+}
contact: Jan Huwald // Impressum